netdevice.h 159.0 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0-or-later */
L
Linus Torvalds 已提交
2 3 4 5 6 7 8 9 10
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		Definitions for the Interfaces handler.
 *
 * Version:	@(#)dev.h	1.0.10	08/12/93
 *
11
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
12 13 14
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Corey Minyard <wf-rch!minyard@relay.EU.net>
 *		Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
15
 *		Alan Cox, <alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
16 17 18 19 20 21 22 23
 *		Bjorn Ekwall. <bj0rn@blox.se>
 *              Pekka Riikonen <priikone@poseidon.pspt.fi>
 *
 *		Moved to /usr/include/linux for NET3
 */
#ifndef _LINUX_NETDEVICE_H
#define _LINUX_NETDEVICE_H

A
Al Viro 已提交
24
#include <linux/timer.h>
25
#include <linux/bug.h>
26
#include <linux/delay.h>
A
Arun Sharma 已提交
27
#include <linux/atomic.h>
28
#include <linux/prefetch.h>
L
Linus Torvalds 已提交
29 30 31 32
#include <asm/cache.h>
#include <asm/byteorder.h>

#include <linux/percpu.h>
33
#include <linux/rculist.h>
34
#include <linux/workqueue.h>
T
Tom Herbert 已提交
35
#include <linux/dynamic_queue_limits.h>
L
Linus Torvalds 已提交
36

37
#include <linux/ethtool.h>
38
#include <net/net_namespace.h>
J
Jeff Kirsher 已提交
39
#ifdef CONFIG_DCB
40 41
#include <net/dcbnl.h>
#endif
42
#include <net/netprio_cgroup.h>
43
#include <net/xdp.h>
44

45
#include <linux/netdev_features.h>
46
#include <linux/neighbour.h>
47
#include <uapi/linux/netdevice.h>
48
#include <uapi/linux/if_bonding.h>
49
#include <uapi/linux/pkt_cls.h>
50
#include <linux/hashtable.h>
51

52
struct netpoll_info;
53
struct device;
54
struct phy_device;
55
struct dsa_port;
56
struct ip_tunnel_parm;
57 58
struct macsec_context;
struct macsec_ops;
59

60
struct sfp_bus;
61 62
/* 802.11 specific */
struct wireless_dev;
63 64
/* 802.15.4 specific */
struct wpan_dev;
R
Robert Shearman 已提交
65
struct mpls_dev;
66 67
/* UDP Tunnel offloads */
struct udp_tunnel_info;
68 69
struct udp_tunnel_nic_info;
struct udp_tunnel_nic;
70
struct bpf_prog;
71
struct xdp_buff;
L
Linus Torvalds 已提交
72

73
void synchronize_net(void);
74 75
void netdev_set_default_ethtool_ops(struct net_device *dev,
				    const struct ethtool_ops *ops);
76

77 78 79 80
/* Backlog congestion levels */
#define NET_RX_SUCCESS		0	/* keep 'em coming, baby */
#define NET_RX_DROP		1	/* packet dropped */

81 82
#define MAX_NEST_DEV 8

83 84 85 86 87 88 89 90 91 92 93 94
/*
 * Transmit return codes: transmit return codes originate from three different
 * namespaces:
 *
 * - qdisc return codes
 * - driver transmit return codes
 * - errno values
 *
 * Drivers are allowed to return any one of those in their hard_start_xmit()
 * function. Real network devices commonly used with qdiscs should only return
 * the driver transmit return codes though - when qdiscs are used, the actual
 * transmission happens asynchronously, so the value is not propagated to
B
Bjorn Helgaas 已提交
95 96
 * higher layers. Virtual network devices transmit synchronously; in this case
 * the driver transmit return codes are consumed by dev_queue_xmit(), and all
97 98 99 100 101
 * others are propagated to higher layers.
 */

/* qdisc ->enqueue() return codes. */
#define NET_XMIT_SUCCESS	0x00
102 103 104
#define NET_XMIT_DROP		0x01	/* skb dropped			*/
#define NET_XMIT_CN		0x02	/* congestion notification	*/
#define NET_XMIT_MASK		0x0f	/* qdisc flags in net/sch_generic.h */
L
Linus Torvalds 已提交
105

106 107 108
/* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
 * indicates that the device will soon be dropping packets, or already drops
 * some packets of the same priority; prompting us to send less aggressively. */
109
#define net_xmit_eval(e)	((e) == NET_XMIT_CN ? 0 : (e))
L
Linus Torvalds 已提交
110 111
#define net_xmit_errno(e)	((e) != NET_XMIT_CN ? -ENOBUFS : 0)

112
/* Driver transmit return codes */
113
#define NETDEV_TX_MASK		0xf0
114

115
enum netdev_tx {
116
	__NETDEV_TX_MIN	 = INT_MIN,	/* make sure enum is signed */
117 118
	NETDEV_TX_OK	 = 0x00,	/* driver took care of packet */
	NETDEV_TX_BUSY	 = 0x10,	/* driver tx path was busy*/
119 120 121
};
typedef enum netdev_tx netdev_tx_t;

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
/*
 * Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
 * hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
 */
static inline bool dev_xmit_complete(int rc)
{
	/*
	 * Positive cases with an skb consumed by a driver:
	 * - successful transmission (rc == NETDEV_TX_OK)
	 * - error while transmitting (rc < 0)
	 * - error while queueing to a different device (rc & NET_XMIT_MASK)
	 */
	if (likely(rc < NET_XMIT_MASK))
		return true;

	return false;
}

L
Linus Torvalds 已提交
140
/*
B
Bjorn Helgaas 已提交
141
 *	Compute the worst-case header length according to the protocols
L
Linus Torvalds 已提交
142 143
 *	used.
 */
G
Graf Yang 已提交
144

145 146 147
#if defined(CONFIG_HYPERV_NET)
# define LL_MAX_HEADER 128
#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
148 149 150 151 152
# if defined(CONFIG_MAC80211_MESH)
#  define LL_MAX_HEADER 128
# else
#  define LL_MAX_HEADER 96
# endif
L
Linus Torvalds 已提交
153
#else
154
# define LL_MAX_HEADER 32
L
Linus Torvalds 已提交
155 156
#endif

157 158
#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
    !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
L
Linus Torvalds 已提交
159 160 161 162 163 164
#define MAX_HEADER LL_MAX_HEADER
#else
#define MAX_HEADER (LL_MAX_HEADER + 48)
#endif

/*
165 166
 *	Old network device statistics. Fields are native words
 *	(unsigned long) so they can be read and written atomically.
L
Linus Torvalds 已提交
167
 */
G
Graf Yang 已提交
168

E
Eric Dumazet 已提交
169
struct net_device_stats {
170 171 172 173 174 175 176 177 178
	unsigned long	rx_packets;
	unsigned long	tx_packets;
	unsigned long	rx_bytes;
	unsigned long	tx_bytes;
	unsigned long	rx_errors;
	unsigned long	tx_errors;
	unsigned long	rx_dropped;
	unsigned long	tx_dropped;
	unsigned long	multicast;
L
Linus Torvalds 已提交
179 180
	unsigned long	collisions;
	unsigned long	rx_length_errors;
181 182 183 184 185
	unsigned long	rx_over_errors;
	unsigned long	rx_crc_errors;
	unsigned long	rx_frame_errors;
	unsigned long	rx_fifo_errors;
	unsigned long	rx_missed_errors;
L
Linus Torvalds 已提交
186 187 188 189 190 191 192 193 194 195 196 197 198
	unsigned long	tx_aborted_errors;
	unsigned long	tx_carrier_errors;
	unsigned long	tx_fifo_errors;
	unsigned long	tx_heartbeat_errors;
	unsigned long	tx_window_errors;
	unsigned long	rx_compressed;
	unsigned long	tx_compressed;
};


#include <linux/cache.h>
#include <linux/skbuff.h>

199
#ifdef CONFIG_RPS
200
#include <linux/static_key.h>
201 202
extern struct static_key_false rps_needed;
extern struct static_key_false rfs_needed;
203 204
#endif

L
Linus Torvalds 已提交
205 206 207 208
struct neighbour;
struct neigh_parms;
struct sk_buff;

209 210 211 212
struct netdev_hw_addr {
	struct list_head	list;
	unsigned char		addr[MAX_ADDR_LEN];
	unsigned char		type;
J
Jiri Pirko 已提交
213 214
#define NETDEV_HW_ADDR_T_LAN		1
#define NETDEV_HW_ADDR_T_SAN		2
T
Taehee Yoo 已提交
215 216
#define NETDEV_HW_ADDR_T_UNICAST	3
#define NETDEV_HW_ADDR_T_MULTICAST	4
217
	bool			global_use;
218
	int			sync_cnt;
219
	int			refcount;
220
	int			synced;
221 222 223
	struct rcu_head		rcu_head;
};

224 225 226 227 228
struct netdev_hw_addr_list {
	struct list_head	list;
	int			count;
};

229 230 231 232
#define netdev_hw_addr_list_count(l) ((l)->count)
#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
#define netdev_hw_addr_list_for_each(ha, l) \
	list_for_each_entry(ha, &(l)->list, list)
233

234 235 236 237
#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
#define netdev_for_each_uc_addr(ha, dev) \
	netdev_hw_addr_list_for_each(ha, &(dev)->uc)
238

239 240
#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
241
#define netdev_for_each_mc_addr(ha, dev) \
242
	netdev_hw_addr_list_for_each(ha, &(dev)->mc)
243

E
Eric Dumazet 已提交
244
struct hh_cache {
245
	unsigned int	hh_len;
246
	seqlock_t	hh_lock;
L
Linus Torvalds 已提交
247 248 249 250

	/* cached hardware header; allow for machine alignment needs.        */
#define HH_DATA_MOD	16
#define HH_DATA_OFF(__len) \
J
Jiri Benc 已提交
251
	(HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
L
Linus Torvalds 已提交
252 253 254 255 256
#define HH_DATA_ALIGN(__len) \
	(((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
	unsigned long	hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
};

B
Bjorn Helgaas 已提交
257
/* Reserve HH_DATA_MOD byte-aligned hard_header_len, but at least that much.
L
Linus Torvalds 已提交
258 259 260 261 262 263 264 265
 * Alternative is:
 *   dev->hard_header_len ? (dev->hard_header_len +
 *                           (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
 *
 * We could use other alignment values, but we must maintain the
 * relationship HH alignment <= LL alignment.
 */
#define LL_RESERVED_SPACE(dev) \
266
	((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
L
Linus Torvalds 已提交
267
#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
268
	((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
L
Linus Torvalds 已提交
269

270 271 272
struct header_ops {
	int	(*create) (struct sk_buff *skb, struct net_device *dev,
			   unsigned short type, const void *daddr,
273
			   const void *saddr, unsigned int len);
274
	int	(*parse)(const struct sk_buff *skb, unsigned char *haddr);
275
	int	(*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
276 277 278
	void	(*cache_update)(struct hh_cache *hh,
				const struct net_device *dev,
				const unsigned char *haddr);
279
	bool	(*validate)(const char *ll_header, unsigned int len);
280
	__be16	(*parse_protocol)(const struct sk_buff *skb);
281 282
};

L
Linus Torvalds 已提交
283
/* These flag bits are private to the generic network queueing
B
Bjorn Helgaas 已提交
284
 * layer; they may not be explicitly referenced by any other
L
Linus Torvalds 已提交
285 286 287
 * code.
 */

E
Eric Dumazet 已提交
288
enum netdev_state_t {
L
Linus Torvalds 已提交
289 290 291
	__LINK_STATE_START,
	__LINK_STATE_PRESENT,
	__LINK_STATE_NOCARRIER,
S
Stefan Rompf 已提交
292 293
	__LINK_STATE_LINKWATCH_PENDING,
	__LINK_STATE_DORMANT,
A
Andrew Lunn 已提交
294
	__LINK_STATE_TESTING,
L
Linus Torvalds 已提交
295 296 297 298
};


/*
B
Bjorn Helgaas 已提交
299
 * This structure holds boot-time configured netdevice settings. They
G
Graf Yang 已提交
300
 * are then used in the device probing.
L
Linus Torvalds 已提交
301 302 303 304 305 306 307
 */
struct netdev_boot_setup {
	char name[IFNAMSIZ];
	struct ifmap map;
};
#define NETDEV_BOOT_SETUP_MAX 8

308
int __init netdev_boot_setup(char *str);
L
Linus Torvalds 已提交
309

310 311 312 313 314
struct gro_list {
	struct list_head	list;
	int			count;
};

315
/*
L
Li RongQing 已提交
316 317
 * size of gro hash buckets, must less than bit number of
 * napi_struct::gro_bitmask
318
 */
319
#define GRO_HASH_BUCKETS	8
L
Li RongQing 已提交
320 321 322 323

/*
 * Structure for NAPI scheduling similar to tasklet but with weighting
 */
324 325 326 327
struct napi_struct {
	/* The poll_list must only be managed by the entity which
	 * changes the state of the NAPI_STATE_SCHED bit.  This means
	 * whoever atomically sets that bit can add this napi_struct
B
Bjorn Helgaas 已提交
328
	 * to the per-CPU poll_list, and whoever clears that bit
329 330 331 332 333 334
	 * can remove from the list right before clearing the bit.
	 */
	struct list_head	poll_list;

	unsigned long		state;
	int			weight;
335
	int			defer_hard_irqs_count;
L
Li RongQing 已提交
336
	unsigned long		gro_bitmask;
337 338 339 340
	int			(*poll)(struct napi_struct *, int);
#ifdef CONFIG_NETPOLL
	int			poll_owner;
#endif
H
Herbert Xu 已提交
341
	struct net_device	*dev;
342
	struct gro_list		gro_hash[GRO_HASH_BUCKETS];
H
Herbert Xu 已提交
343
	struct sk_buff		*skb;
344 345
	struct list_head	rx_list; /* Pending GRO_NORMAL skbs */
	int			rx_count; /* length of rx_list */
346
	struct hrtimer		timer;
347
	struct list_head	dev_list;
E
Eliezer Tamir 已提交
348 349
	struct hlist_node	napi_hash_node;
	unsigned int		napi_id;
350 351
};

E
Eric Dumazet 已提交
352
enum {
353
	NAPI_STATE_SCHED,	/* Poll is scheduled */
E
Eric Dumazet 已提交
354
	NAPI_STATE_MISSED,	/* reschedule a napi */
D
David S. Miller 已提交
355
	NAPI_STATE_DISABLE,	/* Disable pending */
356
	NAPI_STATE_NPSVC,	/* Netpoll - don't dequeue from poll_list */
357
	NAPI_STATE_LISTED,	/* NAPI added to system lists */
E
Eric Dumazet 已提交
358
	NAPI_STATE_NO_BUSY_POLL,/* Do not add in napi_hash, no busy polling */
359 360 361 362
	NAPI_STATE_IN_BUSY_POLL,/* sk_busy_loop() owns this NAPI */
};

enum {
E
Eric Dumazet 已提交
363 364 365 366
	NAPIF_STATE_SCHED	 = BIT(NAPI_STATE_SCHED),
	NAPIF_STATE_MISSED	 = BIT(NAPI_STATE_MISSED),
	NAPIF_STATE_DISABLE	 = BIT(NAPI_STATE_DISABLE),
	NAPIF_STATE_NPSVC	 = BIT(NAPI_STATE_NPSVC),
367
	NAPIF_STATE_LISTED	 = BIT(NAPI_STATE_LISTED),
E
Eric Dumazet 已提交
368 369
	NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
	NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
370 371
};

372
enum gro_result {
373 374 375 376 377
	GRO_MERGED,
	GRO_MERGED_FREE,
	GRO_HELD,
	GRO_NORMAL,
	GRO_DROP,
378
	GRO_CONSUMED,
379
};
380
typedef enum gro_result gro_result_t;
381

382 383 384 385 386 387 388
/*
 * enum rx_handler_result - Possible return values for rx_handlers.
 * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
 * further.
 * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
 * case skb->dev was changed by rx_handler.
 * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
B
Bjorn Helgaas 已提交
389
 * @RX_HANDLER_PASS: Do nothing, pass the skb as if no rx_handler was called.
390 391 392 393 394 395 396 397 398 399 400 401 402 403
 *
 * rx_handlers are functions called from inside __netif_receive_skb(), to do
 * special processing of the skb, prior to delivery to protocol handlers.
 *
 * Currently, a net_device can only have a single rx_handler registered. Trying
 * to register a second rx_handler will return -EBUSY.
 *
 * To register a rx_handler on a net_device, use netdev_rx_handler_register().
 * To unregister a rx_handler on a net_device, use
 * netdev_rx_handler_unregister().
 *
 * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
 * do with the skb.
 *
B
Bjorn Helgaas 已提交
404
 * If the rx_handler consumed the skb in some way, it should return
405
 * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
B
Bjorn Helgaas 已提交
406
 * the skb to be delivered in some other way.
407 408 409 410 411
 *
 * If the rx_handler changed skb->dev, to divert the skb to another
 * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
 * new device will be called if it exists.
 *
B
Bjorn Helgaas 已提交
412
 * If the rx_handler decides the skb should be ignored, it should return
413
 * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
414
 * are registered on exact device (ptype->dev == skb->dev).
415
 *
B
Bjorn Helgaas 已提交
416
 * If the rx_handler didn't change skb->dev, but wants the skb to be normally
417 418 419 420 421 422 423 424 425 426 427 428 429 430
 * delivered, it should return RX_HANDLER_PASS.
 *
 * A device without a registered rx_handler will behave as if rx_handler
 * returned RX_HANDLER_PASS.
 */

enum rx_handler_result {
	RX_HANDLER_CONSUMED,
	RX_HANDLER_ANOTHER,
	RX_HANDLER_EXACT,
	RX_HANDLER_PASS,
};
typedef enum rx_handler_result rx_handler_result_t;
typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
431

432
void __napi_schedule(struct napi_struct *n);
433
void __napi_schedule_irqoff(struct napi_struct *n);
434

435
static inline bool napi_disable_pending(struct napi_struct *n)
D
David S. Miller 已提交
436 437 438 439
{
	return test_bit(NAPI_STATE_DISABLE, &n->state);
}

E
Eric Dumazet 已提交
440
bool napi_schedule_prep(struct napi_struct *n);
441 442 443

/**
 *	napi_schedule - schedule NAPI poll
B
Bjorn Helgaas 已提交
444
 *	@n: NAPI context
445 446 447 448 449 450 451 452 453 454
 *
 * Schedule NAPI poll routine to be called if it is not already
 * running.
 */
static inline void napi_schedule(struct napi_struct *n)
{
	if (napi_schedule_prep(n))
		__napi_schedule(n);
}

455 456
/**
 *	napi_schedule_irqoff - schedule NAPI poll
B
Bjorn Helgaas 已提交
457
 *	@n: NAPI context
458 459 460 461 462 463 464 465 466
 *
 * Variant of napi_schedule(), assuming hard irqs are masked.
 */
static inline void napi_schedule_irqoff(struct napi_struct *n)
{
	if (napi_schedule_prep(n))
		__napi_schedule_irqoff(n);
}

467
/* Try to reschedule poll. Called by dev->poll() after napi_complete().  */
468
static inline bool napi_reschedule(struct napi_struct *napi)
469 470 471
{
	if (napi_schedule_prep(napi)) {
		__napi_schedule(napi);
472
		return true;
473
	}
474
	return false;
475 476
}

477
bool napi_complete_done(struct napi_struct *n, int work_done);
478 479
/**
 *	napi_complete - NAPI processing complete
B
Bjorn Helgaas 已提交
480
 *	@n: NAPI context
481 482
 *
 * Mark NAPI processing as complete.
483
 * Consider using napi_complete_done() instead.
484
 * Return false if device should avoid rearming interrupts.
485
 */
486
static inline bool napi_complete(struct napi_struct *n)
487 488 489
{
	return napi_complete_done(n, 0);
}
490 491 492

/**
 *	napi_disable - prevent NAPI from scheduling
B
Bjorn Helgaas 已提交
493
 *	@n: NAPI context
494 495 496 497
 *
 * Stop NAPI from being scheduled on this context.
 * Waits till any outstanding processing completes.
 */
498
void napi_disable(struct napi_struct *n);
499 500 501

/**
 *	napi_enable - enable NAPI scheduling
B
Bjorn Helgaas 已提交
502
 *	@n: NAPI context
503 504 505 506 507 508 509
 *
 * Resume NAPI from being scheduled on this context.
 * Must be paired with napi_disable.
 */
static inline void napi_enable(struct napi_struct *n)
{
	BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
510
	smp_mb__before_atomic();
511
	clear_bit(NAPI_STATE_SCHED, &n->state);
512
	clear_bit(NAPI_STATE_NPSVC, &n->state);
513 514
}

515 516
/**
 *	napi_synchronize - wait until NAPI is not running
B
Bjorn Helgaas 已提交
517
 *	@n: NAPI context
518 519 520 521 522 523 524
 *
 * Wait until NAPI is done being scheduled on this context.
 * Waits till any outstanding processing completes but
 * does not disable future activations.
 */
static inline void napi_synchronize(const struct napi_struct *n)
{
525 526 527 528 529
	if (IS_ENABLED(CONFIG_SMP))
		while (test_bit(NAPI_STATE_SCHED, &n->state))
			msleep(1);
	else
		barrier();
530 531
}

532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
/**
 *	napi_if_scheduled_mark_missed - if napi is running, set the
 *	NAPIF_STATE_MISSED
 *	@n: NAPI context
 *
 * If napi is running, set the NAPIF_STATE_MISSED, and return true if
 * NAPI is scheduled.
 **/
static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
{
	unsigned long val, new;

	do {
		val = READ_ONCE(n->state);
		if (val & NAPIF_STATE_DISABLE)
			return true;

		if (!(val & NAPIF_STATE_SCHED))
			return false;

		new = val | NAPIF_STATE_MISSED;
	} while (cmpxchg(&n->state, val, new) != val);

	return true;
}

E
Eric Dumazet 已提交
558
enum netdev_queue_state_t {
559 560
	__QUEUE_STATE_DRV_XOFF,
	__QUEUE_STATE_STACK_XOFF,
561
	__QUEUE_STATE_FROZEN,
562
};
563 564 565 566 567 568 569 570 571 572 573

#define QUEUE_STATE_DRV_XOFF	(1 << __QUEUE_STATE_DRV_XOFF)
#define QUEUE_STATE_STACK_XOFF	(1 << __QUEUE_STATE_STACK_XOFF)
#define QUEUE_STATE_FROZEN	(1 << __QUEUE_STATE_FROZEN)

#define QUEUE_STATE_ANY_XOFF	(QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
					QUEUE_STATE_FROZEN)
#define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
					QUEUE_STATE_FROZEN)

574 575 576 577 578 579 580 581 582
/*
 * __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue.  The
 * netif_tx_* functions below are used to manipulate this flag.  The
 * __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
 * queue independently.  The netif_xmit_*stopped functions below are called
 * to check if the queue has been stopped by the driver or stack (either
 * of the XOFF bits are set in the state).  Drivers should not need to call
 * netif_xmit*stopped functions, they should only be using netif_tx_*.
 */
583

584
struct netdev_queue {
585
/*
B
Bjorn Helgaas 已提交
586
 * read-mostly part
587
 */
588
	struct net_device	*dev;
589
	struct Qdisc __rcu	*qdisc;
590
	struct Qdisc		*qdisc_sleeping;
591
#ifdef CONFIG_SYSFS
T
Tom Herbert 已提交
592 593
	struct kobject		kobj;
#endif
594 595 596
#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
	int			numa_node;
#endif
597 598 599 600 601 602
	unsigned long		tx_maxrate;
	/*
	 * Number of TX timeouts for this queue
	 * (/sys/class/net/DEV/Q/trans_timeout)
	 */
	unsigned long		trans_timeout;
603 604 605

	/* Subordinate device that the queue has been assigned to */
	struct net_device	*sb_dev;
606
#ifdef CONFIG_XDP_SOCKETS
607
	struct xsk_buff_pool    *pool;
608
#endif
609
/*
B
Bjorn Helgaas 已提交
610
 * write-mostly part
611 612 613
 */
	spinlock_t		_xmit_lock ____cacheline_aligned_in_smp;
	int			xmit_lock_owner;
614
	/*
F
Florian Westphal 已提交
615
	 * Time (in jiffies) of last Tx
616 617
	 */
	unsigned long		trans_start;
618

T
Tom Herbert 已提交
619 620 621 622 623
	unsigned long		state;

#ifdef CONFIG_BQL
	struct dql		dql;
#endif
624
} ____cacheline_aligned_in_smp;
625

626
extern int sysctl_fb_tunnels_only_for_init_net;
627
extern int sysctl_devconf_inherit_init_net;
628

629 630 631 632 633
/*
 * sysctl_fb_tunnels_only_for_init_net == 0 : For all netns
 *                                     == 1 : For initns only
 *                                     == 2 : For none.
 */
634 635
static inline bool net_has_fallback_tunnels(const struct net *net)
{
636 637 638
	return !IS_ENABLED(CONFIG_SYSCTL) ||
	       !sysctl_fb_tunnels_only_for_init_net ||
	       (net == &init_net && sysctl_fb_tunnels_only_for_init_net == 1);
639 640
}

641 642 643 644 645
static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
{
#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
	return q->numa_node;
#else
646
	return NUMA_NO_NODE;
647 648 649 650 651 652 653 654 655 656
#endif
}

static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
{
#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
	q->numa_node = node;
#endif
}

E
Eric Dumazet 已提交
657
#ifdef CONFIG_RPS
T
Tom Herbert 已提交
658 659 660 661 662 663 664
/*
 * This structure holds an RPS map which can be of variable length.  The
 * map is an array of CPUs.
 */
struct rps_map {
	unsigned int len;
	struct rcu_head rcu;
665
	u16 cpus[];
T
Tom Herbert 已提交
666
};
667
#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
T
Tom Herbert 已提交
668

T
Tom Herbert 已提交
669
/*
670 671 672
 * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
 * tail pointer for that CPU's input queue at the time of last enqueue, and
 * a hardware filter index.
T
Tom Herbert 已提交
673 674 675
 */
struct rps_dev_flow {
	u16 cpu;
676
	u16 filter;
T
Tom Herbert 已提交
677 678
	unsigned int last_qtail;
};
679
#define RPS_NO_FILTER 0xffff
T
Tom Herbert 已提交
680 681 682 683 684 685 686

/*
 * The rps_dev_flow_table structure contains a table of flow mappings.
 */
struct rps_dev_flow_table {
	unsigned int mask;
	struct rcu_head rcu;
687
	struct rps_dev_flow flows[];
T
Tom Herbert 已提交
688 689
};
#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
690
    ((_num) * sizeof(struct rps_dev_flow)))
T
Tom Herbert 已提交
691 692 693 694

/*
 * The rps_sock_flow_table contains mappings of flows to the last CPU
 * on which they were processed by the application (set in recvmsg).
B
Bjorn Helgaas 已提交
695 696
 * Each entry is a 32bit value. Upper part is the high-order bits
 * of flow hash, lower part is CPU number.
697
 * rps_cpu_mask is used to partition the space, depending on number of
B
Bjorn Helgaas 已提交
698 699
 * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
 * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f,
700
 * meaning we use 32-6=26 bits for the hash.
T
Tom Herbert 已提交
701 702
 */
struct rps_sock_flow_table {
703
	u32	mask;
704

705
	u32	ents[] ____cacheline_aligned_in_smp;
T
Tom Herbert 已提交
706
};
707
#define	RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
T
Tom Herbert 已提交
708 709 710

#define RPS_NO_CPU 0xffff

711 712 713
extern u32 rps_cpu_mask;
extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;

T
Tom Herbert 已提交
714 715 716 717
static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
					u32 hash)
{
	if (table && hash) {
718 719
		unsigned int index = hash & table->mask;
		u32 val = hash & ~rps_cpu_mask;
T
Tom Herbert 已提交
720

B
Bjorn Helgaas 已提交
721
		/* We only give a hint, preemption can change CPU under us */
722
		val |= raw_smp_processor_id();
T
Tom Herbert 已提交
723

724 725
		if (table->ents[index] != val)
			table->ents[index] = val;
T
Tom Herbert 已提交
726 727 728
	}
}

729
#ifdef CONFIG_RFS_ACCEL
730 731
bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
			 u16 filter_id);
732
#endif
733
#endif /* CONFIG_RPS */
734

T
Tom Herbert 已提交
735 736
/* This structure contains an instance of an RX queue. */
struct netdev_rx_queue {
737
#ifdef CONFIG_RPS
E
Eric Dumazet 已提交
738 739
	struct rps_map __rcu		*rps_map;
	struct rps_dev_flow_table __rcu	*rps_flow_table;
740
#endif
E
Eric Dumazet 已提交
741
	struct kobject			kobj;
T
Tom Herbert 已提交
742
	struct net_device		*dev;
743
	struct xdp_rxq_info		xdp_rxq;
744
#ifdef CONFIG_XDP_SOCKETS
745
	struct xsk_buff_pool            *pool;
746
#endif
T
Tom Herbert 已提交
747
} ____cacheline_aligned_in_smp;
748 749 750 751 752 753

/*
 * RX queue sysfs structures and functions.
 */
struct rx_queue_attribute {
	struct attribute attr;
754
	ssize_t (*show)(struct netdev_rx_queue *queue, char *buf);
755
	ssize_t (*store)(struct netdev_rx_queue *queue,
756
			 const char *buf, size_t len);
757
};
758

T
Tom Herbert 已提交
759 760 761 762 763 764 765 766 767
#ifdef CONFIG_XPS
/*
 * This structure holds an XPS map which can be of variable length.  The
 * map is an array of queues.
 */
struct xps_map {
	unsigned int len;
	unsigned int alloc_len;
	struct rcu_head rcu;
768
	u16 queues[];
T
Tom Herbert 已提交
769
};
770
#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
771 772
#define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
       - sizeof(struct xps_map)) / sizeof(u16))
T
Tom Herbert 已提交
773 774 775 776 777 778

/*
 * This structure holds all XPS maps for device.  Maps are indexed by CPU.
 */
struct xps_dev_maps {
	struct rcu_head rcu;
779
	struct xps_map __rcu *attr_map[]; /* Either CPUs map or RXQs map */
T
Tom Herbert 已提交
780
};
781 782

#define XPS_CPU_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) +	\
783
	(nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
784 785 786 787

#define XPS_RXQ_DEV_MAPS_SIZE(_tcs, _rxqs) (sizeof(struct xps_dev_maps) +\
	(_rxqs * (_tcs) * sizeof(struct xps_map *)))

T
Tom Herbert 已提交
788 789
#endif /* CONFIG_XPS */

790 791 792 793 794 795 796 797
#define TC_MAX_QUEUE	16
#define TC_BITMASK	15
/* HW offloaded queuing disciplines txq count and offset maps */
struct netdev_tc_txq {
	u16 count;
	u16 offset;
};

798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
/*
 * This structure is to hold information about the device
 * configured to run FCoE protocol stack.
 */
struct netdev_fcoe_hbainfo {
	char	manufacturer[64];
	char	serial_number[64];
	char	hardware_version[64];
	char	driver_version[64];
	char	optionrom_version[64];
	char	firmware_version[64];
	char	model[256];
	char	model_description[256];
};
#endif

815
#define MAX_PHYS_ITEM_ID_LEN 32
816

817 818
/* This structure holds a unique identifier to identify some
 * physical item (port for example) used by a netdevice.
819
 */
820 821
struct netdev_phys_item_id {
	unsigned char id[MAX_PHYS_ITEM_ID_LEN];
822 823 824
	unsigned char id_len;
};

825 826 827 828 829 830 831
static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
					    struct netdev_phys_item_id *b)
{
	return a->id_len == b->id_len &&
	       memcmp(a->id, b->id, a->id_len) == 0;
}

832
typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
833 834
				       struct sk_buff *skb,
				       struct net_device *sb_dev);
835

836
enum tc_setup_type {
837
	TC_SETUP_QDISC_MQPRIO,
838
	TC_SETUP_CLSU32,
839
	TC_SETUP_CLSFLOWER,
840
	TC_SETUP_CLSMATCHALL,
841
	TC_SETUP_CLSBPF,
842
	TC_SETUP_BLOCK,
843
	TC_SETUP_QDISC_CBS,
844
	TC_SETUP_QDISC_RED,
845
	TC_SETUP_QDISC_PRIO,
846
	TC_SETUP_QDISC_MQ,
847
	TC_SETUP_QDISC_ETF,
848
	TC_SETUP_ROOT_QDISC,
849
	TC_SETUP_QDISC_GRED,
850
	TC_SETUP_QDISC_TAPRIO,
851
	TC_SETUP_FT,
852
	TC_SETUP_QDISC_ETS,
853
	TC_SETUP_QDISC_TBF,
854
	TC_SETUP_QDISC_FIFO,
855 856
};

857 858
/* These structures hold the attributes of bpf state that are being passed
 * to the netdevice through the bpf op.
859
 */
860
enum bpf_netdev_command {
861 862 863 864 865 866 867 868
	/* Set or clear a bpf program used in the earliest stages of packet
	 * rx. The prog will have been loaded as BPF_PROG_TYPE_XDP. The callee
	 * is responsible for calling bpf_prog_put on any old progs that are
	 * stored. In case of error, the callee need not release the new prog
	 * reference, but on success it takes ownership and must bpf_prog_put
	 * when it is no longer used.
	 */
	XDP_SETUP_PROG,
869
	XDP_SETUP_PROG_HW,
870
	/* BPF program for offload callbacks, invoked at program load time. */
871 872
	BPF_OFFLOAD_MAP_ALLOC,
	BPF_OFFLOAD_MAP_FREE,
873
	XDP_SETUP_XSK_POOL,
874 875
};

876
struct bpf_prog_offload_ops;
877
struct netlink_ext_ack;
878
struct xdp_umem;
879
struct xdp_dev_bulk_queue;
880
struct bpf_xdp_link;
881

882 883 884 885 886 887 888 889 890
enum bpf_xdp_mode {
	XDP_MODE_SKB = 0,
	XDP_MODE_DRV = 1,
	XDP_MODE_HW = 2,
	__MAX_XDP_MODE
};

struct bpf_xdp_entity {
	struct bpf_prog *prog;
891
	struct bpf_xdp_link *link;
892
};
893

894 895
struct netdev_bpf {
	enum bpf_netdev_command command;
896 897
	union {
		/* XDP_SETUP_PROG */
898
		struct {
899
			u32 flags;
900 901 902
			struct bpf_prog *prog;
			struct netlink_ext_ack *extack;
		};
903 904 905 906
		/* BPF_OFFLOAD_MAP_ALLOC, BPF_OFFLOAD_MAP_FREE */
		struct {
			struct bpf_offloaded_map *offmap;
		};
907
		/* XDP_SETUP_XSK_POOL */
908
		struct {
909
			struct xsk_buff_pool *pool;
910
			u16 queue_id;
911
		} xsk;
912 913
	};
};
914

915 916 917 918
/* Flags for ndo_xsk_wakeup. */
#define XDP_WAKEUP_RX (1 << 0)
#define XDP_WAKEUP_TX (1 << 1)

919 920 921 922 923 924 925
#ifdef CONFIG_XFRM_OFFLOAD
struct xfrmdev_ops {
	int	(*xdo_dev_state_add) (struct xfrm_state *x);
	void	(*xdo_dev_state_delete) (struct xfrm_state *x);
	void	(*xdo_dev_state_free) (struct xfrm_state *x);
	bool	(*xdo_dev_offload_ok) (struct sk_buff *skb,
				       struct xfrm_state *x);
926
	void	(*xdo_dev_state_advance_esn) (struct xfrm_state *x);
927 928 929
};
#endif

930 931 932 933 934
struct dev_ifalias {
	struct rcu_head rcuhead;
	char ifalias[];
};

935
struct devlink;
936
struct tlsdev_ops;
937

938 939
struct netdev_name_node {
	struct hlist_node hlist;
940
	struct list_head list;
941 942 943 944
	struct net_device *dev;
	const char *name;
};

945 946 947
int netdev_name_node_alt_create(struct net_device *dev, const char *name);
int netdev_name_node_alt_destroy(struct net_device *dev, const char *name);

948 949 950 951 952
struct netdev_net_notifier {
	struct list_head list;
	struct notifier_block *nb;
};

953 954
/*
 * This structure defines the management hooks for network devices.
955 956
 * The following hooks can be defined; unless noted otherwise, they are
 * optional and can be filled with a null pointer.
957 958
 *
 * int (*ndo_init)(struct net_device *dev);
B
Bjorn Helgaas 已提交
959 960 961 962
 *     This function is called once when a network device is registered.
 *     The network device can use this for any late stage initialization
 *     or semantic validation. It can fail with an error code which will
 *     be propagated back to register_netdev.
963 964 965 966 967 968
 *
 * void (*ndo_uninit)(struct net_device *dev);
 *     This function is called when device is unregistered or when registration
 *     fails. It is not called if init fails.
 *
 * int (*ndo_open)(struct net_device *dev);
B
Bjorn Helgaas 已提交
969
 *     This function is called when a network device transitions to the up
970 971 972
 *     state.
 *
 * int (*ndo_stop)(struct net_device *dev);
B
Bjorn Helgaas 已提交
973
 *     This function is called when a network device transitions to the down
974 975
 *     state.
 *
976 977
 * netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
 *                               struct net_device *dev);
978
 *	Called when a packet needs to be transmitted.
979 980 981 982
 *	Returns NETDEV_TX_OK.  Can return NETDEV_TX_BUSY, but you should stop
 *	the queue before that can happen; it's for obsolete devices and weird
 *	corner cases, but the stack really does a non-trivial amount
 *	of useless work if you return NETDEV_TX_BUSY.
B
Bjorn Helgaas 已提交
983
 *	Required; cannot be NULL.
984
 *
985 986 987 988 989 990 991 992 993
 * netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
 *					   struct net_device *dev
 *					   netdev_features_t features);
 *	Called by core transmit path to determine if device is capable of
 *	performing offload operations on a given packet. This is to give
 *	the device an opportunity to implement any restrictions that cannot
 *	be otherwise expressed by feature flags. The check is called with
 *	the set of features that the stack has calculated and it returns
 *	those the driver believes to be appropriate.
994
 *
995
 * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
996
 *                         struct net_device *sb_dev);
B
Bjorn Helgaas 已提交
997
 *	Called to decide which queue to use when device supports multiple
998 999
 *	transmit queues.
 *
1000 1001
 * void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
 *	This function is called to allow device receiver to make
B
Bjorn Helgaas 已提交
1002
 *	changes to configuration when multicast or promiscuous is enabled.
1003 1004 1005
 *
 * void (*ndo_set_rx_mode)(struct net_device *dev);
 *	This function is called device changes address list filtering.
1006
 *	If driver handles unicast address filtering, it should set
B
Bjorn Helgaas 已提交
1007
 *	IFF_UNICAST_FLT in its priv_flags.
1008 1009 1010
 *
 * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
 *	This function  is called when the Media Access Control address
1011
 *	needs to be changed. If this interface is not defined, the
B
Bjorn Helgaas 已提交
1012
 *	MAC address can not be changed.
1013 1014 1015 1016 1017
 *
 * int (*ndo_validate_addr)(struct net_device *dev);
 *	Test if Media Access Control address is valid for the device.
 *
 * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
B
Bjorn Helgaas 已提交
1018 1019
 *	Called when a user requests an ioctl which can't be handled by
 *	the generic interface code. If not defined ioctls return
1020 1021 1022 1023
 *	not supported error code.
 *
 * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
 *	Used to set network devices bus interface parameters. This interface
B
Bjorn Helgaas 已提交
1024
 *	is retained for legacy reasons; new devices should use the bus
1025 1026 1027 1028
 *	interface (PCI) for low level management.
 *
 * int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
 *	Called when a user wants to change the Maximum Transfer Unit
1029
 *	of a device.
1030
 *
1031
 * void (*ndo_tx_timeout)(struct net_device *dev, unsigned int txqueue);
B
Bjorn Helgaas 已提交
1032
 *	Callback used when the transmitter has not made any progress
1033 1034
 *	for dev->watchdog ticks.
 *
1035 1036
 * void (*ndo_get_stats64)(struct net_device *dev,
 *                         struct rtnl_link_stats64 *storage);
1037
 * struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1038
 *	Called when a user wants to get the network device usage
1039
 *	statistics. Drivers must do one of the following:
1040 1041
 *	1. Define @ndo_get_stats64 to fill in a zero-initialised
 *	   rtnl_link_stats64 structure passed by the caller.
1042
 *	2. Define @ndo_get_stats to update a net_device_stats structure
1043 1044 1045 1046 1047
 *	   (which should normally be dev->stats) and return a pointer to
 *	   it. The structure may be changed asynchronously only if each
 *	   field is written atomically.
 *	3. Update dev->stats asynchronously and atomically, and define
 *	   neither operation.
1048
 *
1049
 * bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id)
1050 1051 1052 1053 1054 1055 1056
 *	Return true if this device supports offload stats of this attr_id.
 *
 * int (*ndo_get_offload_stats)(int attr_id, const struct net_device *dev,
 *	void *attr_data)
 *	Get statistics for offload operations by attr_id. Write it into the
 *	attr_data pointer.
 *
1057
 * int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
B
Bjorn Helgaas 已提交
1058
 *	If device supports VLAN filtering this function is called when a
1059
 *	VLAN id is registered.
1060
 *
1061
 * int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
B
Bjorn Helgaas 已提交
1062
 *	If device supports VLAN filtering this function is called when a
1063
 *	VLAN id is unregistered.
1064 1065
 *
 * void (*ndo_poll_controller)(struct net_device *dev);
1066 1067 1068
 *
 *	SR-IOV management functions.
 * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
1069 1070
 * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan,
 *			  u8 qos, __be16 proto);
1071 1072
 * int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
 *			  int max_tx_rate);
1073
 * int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
H
Hiroshi Shimamoto 已提交
1074
 * int (*ndo_set_vf_trust)(struct net_device *dev, int vf, bool setting);
1075 1076
 * int (*ndo_get_vf_config)(struct net_device *dev,
 *			    int vf, struct ifla_vf_info *ivf);
1077
 * int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
1078 1079
 * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
 *			  struct nlattr *port[]);
1080 1081 1082
 *
 *      Enable or disable the VF ability to query its RSS Redirection Table and
 *      Hash Key. This is needed since on some devices VF share this information
B
Bjorn Helgaas 已提交
1083
 *      with PF and querying it may introduce a theoretical security risk.
1084
 * int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
1085
 * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
1086
 * int (*ndo_setup_tc)(struct net_device *dev, enum tc_setup_type type,
1087
 *		       void *type_data);
F
Florian Fainelli 已提交
1088 1089 1090 1091
 *	Called to setup any 'tc' scheduler, classifier or action on @dev.
 *	This is always called from the stack with the rtnl lock held and netif
 *	tx queues stopped. This allows the netdevice to perform queue
 *	management safely.
1092
 *
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
 *	Fiber Channel over Ethernet (FCoE) offload functions.
 * int (*ndo_fcoe_enable)(struct net_device *dev);
 *	Called when the FCoE protocol stack wants to start using LLD for FCoE
 *	so the underlying device can perform whatever needed configuration or
 *	initialization to support acceleration of FCoE traffic.
 *
 * int (*ndo_fcoe_disable)(struct net_device *dev);
 *	Called when the FCoE protocol stack wants to stop using LLD for FCoE
 *	so the underlying device can perform whatever needed clean-ups to
 *	stop supporting acceleration of FCoE traffic.
 *
 * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
 *			     struct scatterlist *sgl, unsigned int sgc);
 *	Called when the FCoE Initiator wants to initialize an I/O that
 *	is a possible candidate for Direct Data Placement (DDP). The LLD can
 *	perform necessary setup and returns 1 to indicate the device is set up
 *	successfully to perform DDP on this I/O, otherwise this returns 0.
 *
 * int (*ndo_fcoe_ddp_done)(struct net_device *dev,  u16 xid);
 *	Called when the FCoE Initiator/Target is done with the DDPed I/O as
 *	indicated by the FC exchange id 'xid', so the underlying device can
 *	clean up and reuse resources for later DDP requests.
 *
 * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
 *			      struct scatterlist *sgl, unsigned int sgc);
 *	Called when the FCoE Target wants to initialize an I/O that
 *	is a possible candidate for Direct Data Placement (DDP). The LLD can
 *	perform necessary setup and returns 1 to indicate the device is set up
 *	successfully to perform DDP on this I/O, otherwise this returns 0.
 *
1123 1124 1125 1126 1127 1128 1129
 * int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
 *			       struct netdev_fcoe_hbainfo *hbainfo);
 *	Called when the FCoE Protocol stack wants information on the underlying
 *	device. This information is utilized by the FCoE protocol stack to
 *	register attributes with Fiber Channel management service as per the
 *	FC-GS Fabric Device Management Information(FDMI) specification.
 *
1130 1131 1132 1133 1134 1135
 * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
 *	Called when the underlying device wants to override default World Wide
 *	Name (WWN) generation mechanism in FCoE protocol stack to pass its own
 *	World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
 *	protocol stack to use.
 *
1136 1137 1138 1139 1140 1141
 *	RFS acceleration.
 * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
 *			    u16 rxq_index, u32 flow_id);
 *	Set hardware filter for RFS.  rxq_index is the target queue index;
 *	flow_id is a flow ID to be passed to rps_may_expire_flow() later.
 *	Return the filter ID on success, or a negative error code.
1142
 *
1143
 *	Slave management functions (for bridge, bonding, etc).
1144 1145 1146 1147 1148
 * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
 *	Called to make another netdev an underling.
 *
 * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
 *	Called to release previously enslaved netdev.
1149
 *
1150 1151 1152 1153 1154 1155
 * struct net_device *(*ndo_get_xmit_slave)(struct net_device *dev,
 *					    struct sk_buff *skb,
 *					    bool all_slaves);
 *	Get the xmit slave of master device. If all_slaves is true, function
 *	assume all the slaves can transmit.
 *
1156
 *      Feature/offload setting functions.
1157 1158 1159 1160 1161 1162
 * netdev_features_t (*ndo_fix_features)(struct net_device *dev,
 *		netdev_features_t features);
 *	Adjusts the requested feature flags according to device-specific
 *	constraints, and returns the resulting flags. Must not modify
 *	the device state.
 *
1163
 * int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
1164 1165 1166 1167
 *	Called to update device configuration to new features. Passed
 *	feature set might be less than what was returned by ndo_fix_features()).
 *	Must return >0 or -errno if it changed dev->features itself.
 *
1168 1169
 * int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
 *		      struct net_device *dev,
1170 1171
 *		      const unsigned char *addr, u16 vid, u16 flags,
 *		      struct netlink_ext_ack *extack);
1172
 *	Adds an FDB entry to dev for addr.
1173 1174
 * int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
 *		      struct net_device *dev,
1175
 *		      const unsigned char *addr, u16 vid)
1176 1177
 *	Deletes the FDB entry from dev coresponding to addr.
 * int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
1178
 *		       struct net_device *dev, struct net_device *filter_dev,
1179
 *		       int *idx)
1180 1181
 *	Used to add FDB entries to dump requests. Implementers should add
 *	entries to skb and update idx with the number of entries.
J
John Fastabend 已提交
1182
 *
1183
 * int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
P
Petr Machata 已提交
1184
 *			     u16 flags, struct netlink_ext_ack *extack)
J
John Fastabend 已提交
1185
 * int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
1186 1187
 *			     struct net_device *dev, u32 filter_mask,
 *			     int nlflags)
1188 1189
 * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
 *			     u16 flags);
J
Jiri Pirko 已提交
1190 1191 1192 1193 1194 1195 1196 1197
 *
 * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
 *	Called to change device carrier. Soft-devices (like dummy, team, etc)
 *	which do not represent real hardware may define this to allow their
 *	userspace components to manage their virtual carrier state. Devices
 *	that determine carrier state from physical hardware properties (eg
 *	network cables) or protocol-dependent mechanisms (eg
 *	USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
1198 1199
 *
 * int (*ndo_get_phys_port_id)(struct net_device *dev,
1200
 *			       struct netdev_phys_item_id *ppid);
1201 1202 1203
 *	Called to get ID of physical port of this device. If driver does
 *	not implement this, it is assumed that the hw is not able to have
 *	multiple net devices on single physical port.
1204
 *
1205 1206 1207 1208
 * int (*ndo_get_port_parent_id)(struct net_device *dev,
 *				 struct netdev_phys_item_id *ppid)
 *	Called to get the parent ID of the physical port of this device.
 *
1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
 * void (*ndo_udp_tunnel_add)(struct net_device *dev,
 *			      struct udp_tunnel_info *ti);
 *	Called by UDP tunnel to notify a driver about the UDP port and socket
 *	address family that a UDP tunnel is listnening to. It is called only
 *	when a new port starts listening. The operation is protected by the
 *	RTNL.
 *
 * void (*ndo_udp_tunnel_del)(struct net_device *dev,
 *			      struct udp_tunnel_info *ti);
 *	Called by UDP tunnel to notify the driver about a UDP port and socket
 *	address family that the UDP tunnel is not listening to anymore. The
 *	operation is protected by the RTNL.
 *
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
 * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
 *				 struct net_device *dev)
 *	Called by upper layer devices to accelerate switching or other
 *	station functionality into hardware. 'pdev is the lowerdev
 *	to use for the offload and 'dev' is the net device that will
 *	back the offload. Returns a pointer to the private structure
 *	the upper layer will maintain.
 * void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
 *	Called by upper layer device to delete the station created
 *	by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
 *	the station and priv is the structure returned by the add
 *	operation.
1234 1235 1236 1237
 * int (*ndo_set_tx_maxrate)(struct net_device *dev,
 *			     int queue_index, u32 maxrate);
 *	Called when a user wants to set a max-rate limitation of specific
 *	TX queue.
1238 1239
 * int (*ndo_get_iflink)(const struct net_device *dev);
 *	Called to get the iflink value of this device.
1240
 * void (*ndo_change_proto_down)(struct net_device *dev,
B
Bjorn Helgaas 已提交
1241
 *				 bool proto_down);
1242 1243 1244
 *	This function is used to pass protocol port error state information
 *	to the switch driver. The switch driver can react to the proto_down
 *      by doing a phys down on the associated switch port.
1245 1246 1247 1248
 * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb);
 *	This function is used to get egress tunnel information for given skb.
 *	This is useful for retrieving outer tunnel header parameters while
 *	sampling packet.
1249 1250 1251 1252
 * void (*ndo_set_rx_headroom)(struct net_device *dev, int needed_headroom);
 *	This function is used to specify the headroom that the skb must
 *	consider when allocation skb during packet reception. Setting
 *	appropriate rx headroom value allows avoiding skb head copy on
B
Bjorn Helgaas 已提交
1253
 *	forward. Setting a negative value resets the rx headroom to the
1254
 *	default value.
1255
 * int (*ndo_bpf)(struct net_device *dev, struct netdev_bpf *bpf);
1256
 *	This function is used to set or query state related to XDP on the
1257 1258
 *	netdevice and manage BPF offload. See definition of
 *	enum bpf_netdev_command for details.
1259 1260
 * int (*ndo_xdp_xmit)(struct net_device *dev, int n, struct xdp_frame **xdp,
 *			u32 flags);
1261 1262 1263 1264 1265
 *	This function is used to submit @n XDP packets for transmit on a
 *	netdevice. Returns number of frames successfully transmitted, frames
 *	that got dropped are freed/returned via xdp_return_frame().
 *	Returns negative number, means general error invoking ndo, meaning
 *	no frames were xmit'ed and core-caller will free all frames.
1266 1267 1268 1269 1270 1271
 * int (*ndo_xsk_wakeup)(struct net_device *dev, u32 queue_id, u32 flags);
 *      This function is used to wake up the softirq, ksoftirqd or kthread
 *	responsible for sending and/or receiving packets on a specific
 *	queue id bound to an AF_XDP socket. The flags field specifies if
 *	only RX, only Tx, or both should be woken up using the flags
 *	XDP_WAKEUP_RX and XDP_WAKEUP_TX.
1272 1273
 * struct devlink_port *(*ndo_get_devlink_port)(struct net_device *dev);
 *	Get devlink port instance associated with a given netdev.
1274 1275
 *	Called with a reference on the netdevice and devlink locks only,
 *	rtnl_lock is not held.
1276 1277 1278
 * int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm *p,
 *			 int cmd);
 *	Add, change, delete or get information on an IPv4 tunnel.
D
Daniel Borkmann 已提交
1279 1280 1281
 * struct net_device *(*ndo_get_peer_dev)(struct net_device *dev);
 *	If a device is paired with a peer device, return the peer instance.
 *	The caller must be under RCU read context.
1282 1283 1284 1285 1286 1287
 */
struct net_device_ops {
	int			(*ndo_init)(struct net_device *dev);
	void			(*ndo_uninit)(struct net_device *dev);
	int			(*ndo_open)(struct net_device *dev);
	int			(*ndo_stop)(struct net_device *dev);
1288 1289 1290 1291 1292
	netdev_tx_t		(*ndo_start_xmit)(struct sk_buff *skb,
						  struct net_device *dev);
	netdev_features_t	(*ndo_features_check)(struct sk_buff *skb,
						      struct net_device *dev,
						      netdev_features_t features);
1293
	u16			(*ndo_select_queue)(struct net_device *dev,
1294
						    struct sk_buff *skb,
1295
						    struct net_device *sb_dev);
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305
	void			(*ndo_change_rx_flags)(struct net_device *dev,
						       int flags);
	void			(*ndo_set_rx_mode)(struct net_device *dev);
	int			(*ndo_set_mac_address)(struct net_device *dev,
						       void *addr);
	int			(*ndo_validate_addr)(struct net_device *dev);
	int			(*ndo_do_ioctl)(struct net_device *dev,
					        struct ifreq *ifr, int cmd);
	int			(*ndo_set_config)(struct net_device *dev,
					          struct ifmap *map);
1306 1307 1308 1309
	int			(*ndo_change_mtu)(struct net_device *dev,
						  int new_mtu);
	int			(*ndo_neigh_setup)(struct net_device *dev,
						   struct neigh_parms *);
1310 1311
	void			(*ndo_tx_timeout) (struct net_device *dev,
						   unsigned int txqueue);
1312

1313 1314
	void			(*ndo_get_stats64)(struct net_device *dev,
						   struct rtnl_link_stats64 *storage);
1315
	bool			(*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1316 1317 1318
	int			(*ndo_get_offload_stats)(int attr_id,
							 const struct net_device *dev,
							 void *attr_data);
1319 1320
	struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);

1321
	int			(*ndo_vlan_rx_add_vid)(struct net_device *dev,
1322
						       __be16 proto, u16 vid);
1323
	int			(*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1324
						        __be16 proto, u16 vid);
1325 1326
#ifdef CONFIG_NET_POLL_CONTROLLER
	void                    (*ndo_poll_controller)(struct net_device *dev);
H
Herbert Xu 已提交
1327
	int			(*ndo_netpoll_setup)(struct net_device *dev,
1328
						     struct netpoll_info *info);
1329
	void			(*ndo_netpoll_cleanup)(struct net_device *dev);
1330
#endif
1331 1332 1333
	int			(*ndo_set_vf_mac)(struct net_device *dev,
						  int queue, u8 *mac);
	int			(*ndo_set_vf_vlan)(struct net_device *dev,
1334 1335
						   int queue, u16 vlan,
						   u8 qos, __be16 proto);
1336 1337 1338
	int			(*ndo_set_vf_rate)(struct net_device *dev,
						   int vf, int min_tx_rate,
						   int max_tx_rate);
1339 1340
	int			(*ndo_set_vf_spoofchk)(struct net_device *dev,
						       int vf, bool setting);
H
Hiroshi Shimamoto 已提交
1341 1342
	int			(*ndo_set_vf_trust)(struct net_device *dev,
						    int vf, bool setting);
1343 1344 1345
	int			(*ndo_get_vf_config)(struct net_device *dev,
						     int vf,
						     struct ifla_vf_info *ivf);
1346 1347
	int			(*ndo_set_vf_link_state)(struct net_device *dev,
							 int vf, int link_state);
1348 1349 1350 1351
	int			(*ndo_get_vf_stats)(struct net_device *dev,
						    int vf,
						    struct ifla_vf_stats
						    *vf_stats);
1352 1353 1354 1355 1356
	int			(*ndo_set_vf_port)(struct net_device *dev,
						   int vf,
						   struct nlattr *port[]);
	int			(*ndo_get_vf_port)(struct net_device *dev,
						   int vf, struct sk_buff *skb);
1357 1358 1359 1360
	int			(*ndo_get_vf_guid)(struct net_device *dev,
						   int vf,
						   struct ifla_vf_guid *node_guid,
						   struct ifla_vf_guid *port_guid);
1361 1362 1363
	int			(*ndo_set_vf_guid)(struct net_device *dev,
						   int vf, u64 guid,
						   int guid_type);
1364 1365 1366
	int			(*ndo_set_vf_rss_query_en)(
						   struct net_device *dev,
						   int vf, bool setting);
1367
	int			(*ndo_setup_tc)(struct net_device *dev,
1368
						enum tc_setup_type type,
1369
						void *type_data);
1370
#if IS_ENABLED(CONFIG_FCOE)
1371 1372
	int			(*ndo_fcoe_enable)(struct net_device *dev);
	int			(*ndo_fcoe_disable)(struct net_device *dev);
1373 1374 1375 1376 1377 1378
	int			(*ndo_fcoe_ddp_setup)(struct net_device *dev,
						      u16 xid,
						      struct scatterlist *sgl,
						      unsigned int sgc);
	int			(*ndo_fcoe_ddp_done)(struct net_device *dev,
						     u16 xid);
1379 1380 1381 1382
	int			(*ndo_fcoe_ddp_target)(struct net_device *dev,
						       u16 xid,
						       struct scatterlist *sgl,
						       unsigned int sgc);
1383 1384
	int			(*ndo_fcoe_get_hbainfo)(struct net_device *dev,
							struct netdev_fcoe_hbainfo *hbainfo);
1385 1386
#endif

1387
#if IS_ENABLED(CONFIG_LIBFCOE)
1388 1389 1390 1391
#define NETDEV_FCOE_WWNN 0
#define NETDEV_FCOE_WWPN 1
	int			(*ndo_fcoe_get_wwn)(struct net_device *dev,
						    u64 *wwn, int type);
1392
#endif
1393

1394 1395 1396 1397 1398 1399
#ifdef CONFIG_RFS_ACCEL
	int			(*ndo_rx_flow_steer)(struct net_device *dev,
						     const struct sk_buff *skb,
						     u16 rxq_index,
						     u32 flow_id);
#endif
1400
	int			(*ndo_add_slave)(struct net_device *dev,
D
David Ahern 已提交
1401 1402
						 struct net_device *slave_dev,
						 struct netlink_ext_ack *extack);
1403 1404
	int			(*ndo_del_slave)(struct net_device *dev,
						 struct net_device *slave_dev);
1405 1406 1407
	struct net_device*	(*ndo_get_xmit_slave)(struct net_device *dev,
						      struct sk_buff *skb,
						      bool all_slaves);
1408 1409
	netdev_features_t	(*ndo_fix_features)(struct net_device *dev,
						    netdev_features_t features);
1410
	int			(*ndo_set_features)(struct net_device *dev,
1411
						    netdev_features_t features);
1412 1413 1414 1415
	int			(*ndo_neigh_construct)(struct net_device *dev,
						       struct neighbour *n);
	void			(*ndo_neigh_destroy)(struct net_device *dev,
						     struct neighbour *n);
1416 1417

	int			(*ndo_fdb_add)(struct ndmsg *ndm,
1418
					       struct nlattr *tb[],
1419
					       struct net_device *dev,
1420
					       const unsigned char *addr,
1421
					       u16 vid,
1422 1423
					       u16 flags,
					       struct netlink_ext_ack *extack);
1424
	int			(*ndo_fdb_del)(struct ndmsg *ndm,
1425
					       struct nlattr *tb[],
1426
					       struct net_device *dev,
1427 1428
					       const unsigned char *addr,
					       u16 vid);
1429 1430 1431
	int			(*ndo_fdb_dump)(struct sk_buff *skb,
						struct netlink_callback *cb,
						struct net_device *dev,
1432
						struct net_device *filter_dev,
1433
						int *idx);
R
Roopa Prabhu 已提交
1434 1435 1436 1437 1438 1439
	int			(*ndo_fdb_get)(struct sk_buff *skb,
					       struct nlattr *tb[],
					       struct net_device *dev,
					       const unsigned char *addr,
					       u16 vid, u32 portid, u32 seq,
					       struct netlink_ext_ack *extack);
J
John Fastabend 已提交
1440
	int			(*ndo_bridge_setlink)(struct net_device *dev,
1441
						      struct nlmsghdr *nlh,
P
Petr Machata 已提交
1442 1443
						      u16 flags,
						      struct netlink_ext_ack *extack);
J
John Fastabend 已提交
1444 1445
	int			(*ndo_bridge_getlink)(struct sk_buff *skb,
						      u32 pid, u32 seq,
1446
						      struct net_device *dev,
1447 1448
						      u32 filter_mask,
						      int nlflags);
1449
	int			(*ndo_bridge_dellink)(struct net_device *dev,
1450 1451
						      struct nlmsghdr *nlh,
						      u16 flags);
J
Jiri Pirko 已提交
1452 1453
	int			(*ndo_change_carrier)(struct net_device *dev,
						      bool new_carrier);
1454
	int			(*ndo_get_phys_port_id)(struct net_device *dev,
1455
							struct netdev_phys_item_id *ppid);
1456 1457
	int			(*ndo_get_port_parent_id)(struct net_device *dev,
							  struct netdev_phys_item_id *ppid);
1458 1459
	int			(*ndo_get_phys_port_name)(struct net_device *dev,
							  char *name, size_t len);
1460 1461 1462 1463
	void			(*ndo_udp_tunnel_add)(struct net_device *dev,
						      struct udp_tunnel_info *ti);
	void			(*ndo_udp_tunnel_del)(struct net_device *dev,
						      struct udp_tunnel_info *ti);
1464 1465 1466 1467 1468
	void*			(*ndo_dfwd_add_station)(struct net_device *pdev,
							struct net_device *dev);
	void			(*ndo_dfwd_del_station)(struct net_device *pdev,
							void *priv);

1469 1470 1471
	int			(*ndo_set_tx_maxrate)(struct net_device *dev,
						      int queue_index,
						      u32 maxrate);
1472
	int			(*ndo_get_iflink)(const struct net_device *dev);
1473 1474
	int			(*ndo_change_proto_down)(struct net_device *dev,
							 bool proto_down);
1475 1476
	int			(*ndo_fill_metadata_dst)(struct net_device *dev,
						       struct sk_buff *skb);
1477 1478
	void			(*ndo_set_rx_headroom)(struct net_device *dev,
						       int needed_headroom);
1479 1480
	int			(*ndo_bpf)(struct net_device *dev,
					   struct netdev_bpf *bpf);
1481
	int			(*ndo_xdp_xmit)(struct net_device *dev, int n,
1482 1483
						struct xdp_frame **xdp,
						u32 flags);
1484 1485
	int			(*ndo_xsk_wakeup)(struct net_device *dev,
						  u32 queue_id, u32 flags);
1486
	struct devlink_port *	(*ndo_get_devlink_port)(struct net_device *dev);
1487 1488
	int			(*ndo_tunnel_ctl)(struct net_device *dev,
						  struct ip_tunnel_parm *p, int cmd);
D
Daniel Borkmann 已提交
1489
	struct net_device *	(*ndo_get_peer_dev)(struct net_device *dev);
1490 1491
};

1492 1493 1494 1495 1496
/**
 * enum net_device_priv_flags - &struct net_device priv_flags
 *
 * These are the &struct net_device, they are only set internally
 * by drivers and used in the kernel. These flags are invisible to
B
Bjorn Helgaas 已提交
1497
 * userspace; this means that the order of these flags can change
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
 * during any kernel release.
 *
 * You should have a pretty good reason to be extending these flags.
 *
 * @IFF_802_1Q_VLAN: 802.1Q VLAN device
 * @IFF_EBRIDGE: Ethernet bridging device
 * @IFF_BONDING: bonding master or slave
 * @IFF_ISATAP: ISATAP interface (RFC4214)
 * @IFF_WAN_HDLC: WAN HDLC device
 * @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
 *	release skb->dst
 * @IFF_DONT_BRIDGE: disallow bridging this ether dev
 * @IFF_DISABLE_NETPOLL: disable netpoll at run-time
 * @IFF_MACVLAN_PORT: device used as macvlan port
 * @IFF_BRIDGE_PORT: device used as bridge port
 * @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
 * @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
 * @IFF_UNICAST_FLT: Supports unicast filtering
 * @IFF_TEAM_PORT: device used as team port
 * @IFF_SUPP_NOFCS: device supports sending custom FCS
 * @IFF_LIVE_ADDR_CHANGE: device supports hardware address
 *	change when it's running
 * @IFF_MACVLAN: Macvlan device
1521 1522
 * @IFF_XMIT_DST_RELEASE_PERM: IFF_XMIT_DST_RELEASE not taking into account
 *	underlying stacked devices
1523
 * @IFF_L3MDEV_MASTER: device is an L3 master device
1524
 * @IFF_NO_QUEUE: device can run without qdisc attached
1525
 * @IFF_OPENVSWITCH: device is a Open vSwitch master
D
David Ahern 已提交
1526
 * @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device
J
Jiri Pirko 已提交
1527
 * @IFF_TEAM: device is a team device
1528
 * @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured
1529 1530
 * @IFF_PHONY_HEADROOM: the headroom value is controlled by an external
 *	entity (i.e. the master device for bridged veth)
1531
 * @IFF_MACSEC: device is a MACsec device
P
Paolo Abeni 已提交
1532
 * @IFF_NO_RX_HANDLER: device doesn't support the rx_handler hook
1533 1534
 * @IFF_FAILOVER: device is a failover master device
 * @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device
1535
 * @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device
1536
 * @IFF_LIVE_RENAME_OK: rename is allowed while device is up and running
1537 1538 1539 1540
 */
enum netdev_priv_flags {
	IFF_802_1Q_VLAN			= 1<<0,
	IFF_EBRIDGE			= 1<<1,
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556
	IFF_BONDING			= 1<<2,
	IFF_ISATAP			= 1<<3,
	IFF_WAN_HDLC			= 1<<4,
	IFF_XMIT_DST_RELEASE		= 1<<5,
	IFF_DONT_BRIDGE			= 1<<6,
	IFF_DISABLE_NETPOLL		= 1<<7,
	IFF_MACVLAN_PORT		= 1<<8,
	IFF_BRIDGE_PORT			= 1<<9,
	IFF_OVS_DATAPATH		= 1<<10,
	IFF_TX_SKB_SHARING		= 1<<11,
	IFF_UNICAST_FLT			= 1<<12,
	IFF_TEAM_PORT			= 1<<13,
	IFF_SUPP_NOFCS			= 1<<14,
	IFF_LIVE_ADDR_CHANGE		= 1<<15,
	IFF_MACVLAN			= 1<<16,
	IFF_XMIT_DST_RELEASE_PERM	= 1<<17,
P
Paolo Abeni 已提交
1557 1558 1559 1560 1561 1562 1563 1564
	IFF_L3MDEV_MASTER		= 1<<18,
	IFF_NO_QUEUE			= 1<<19,
	IFF_OPENVSWITCH			= 1<<20,
	IFF_L3MDEV_SLAVE		= 1<<21,
	IFF_TEAM			= 1<<22,
	IFF_RXFH_CONFIGURED		= 1<<23,
	IFF_PHONY_HEADROOM		= 1<<24,
	IFF_MACSEC			= 1<<25,
P
Paolo Abeni 已提交
1565
	IFF_NO_RX_HANDLER		= 1<<26,
1566 1567
	IFF_FAILOVER			= 1<<27,
	IFF_FAILOVER_SLAVE		= 1<<28,
1568
	IFF_L3MDEV_RX_HANDLER		= 1<<29,
1569
	IFF_LIVE_RENAME_OK		= 1<<30,
1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588
};

#define IFF_802_1Q_VLAN			IFF_802_1Q_VLAN
#define IFF_EBRIDGE			IFF_EBRIDGE
#define IFF_BONDING			IFF_BONDING
#define IFF_ISATAP			IFF_ISATAP
#define IFF_WAN_HDLC			IFF_WAN_HDLC
#define IFF_XMIT_DST_RELEASE		IFF_XMIT_DST_RELEASE
#define IFF_DONT_BRIDGE			IFF_DONT_BRIDGE
#define IFF_DISABLE_NETPOLL		IFF_DISABLE_NETPOLL
#define IFF_MACVLAN_PORT		IFF_MACVLAN_PORT
#define IFF_BRIDGE_PORT			IFF_BRIDGE_PORT
#define IFF_OVS_DATAPATH		IFF_OVS_DATAPATH
#define IFF_TX_SKB_SHARING		IFF_TX_SKB_SHARING
#define IFF_UNICAST_FLT			IFF_UNICAST_FLT
#define IFF_TEAM_PORT			IFF_TEAM_PORT
#define IFF_SUPP_NOFCS			IFF_SUPP_NOFCS
#define IFF_LIVE_ADDR_CHANGE		IFF_LIVE_ADDR_CHANGE
#define IFF_MACVLAN			IFF_MACVLAN
1589
#define IFF_XMIT_DST_RELEASE_PERM	IFF_XMIT_DST_RELEASE_PERM
1590
#define IFF_L3MDEV_MASTER		IFF_L3MDEV_MASTER
1591
#define IFF_NO_QUEUE			IFF_NO_QUEUE
1592
#define IFF_OPENVSWITCH			IFF_OPENVSWITCH
1593
#define IFF_L3MDEV_SLAVE		IFF_L3MDEV_SLAVE
J
Jiri Pirko 已提交
1594
#define IFF_TEAM			IFF_TEAM
1595
#define IFF_RXFH_CONFIGURED		IFF_RXFH_CONFIGURED
1596
#define IFF_MACSEC			IFF_MACSEC
P
Paolo Abeni 已提交
1597
#define IFF_NO_RX_HANDLER		IFF_NO_RX_HANDLER
1598 1599
#define IFF_FAILOVER			IFF_FAILOVER
#define IFF_FAILOVER_SLAVE		IFF_FAILOVER_SLAVE
1600
#define IFF_L3MDEV_RX_HANDLER		IFF_L3MDEV_RX_HANDLER
1601
#define IFF_LIVE_RENAME_OK		IFF_LIVE_RENAME_OK
1602

1603 1604
/**
 *	struct net_device - The DEVICE structure.
1605 1606 1607 1608
 *
 *	Actually, this whole structure is a big mistake.  It mixes I/O
 *	data with strictly "high-level" data, and it has to know about
 *	almost every data structure used in the INET module.
1609 1610 1611
 *
 *	@name:	This is the first field of the "visible" part of this structure
 *		(i.e. as seen by users in the "Space.c" file).  It is the name
1612
 *		of the interface.
1613
 *
1614
 *	@name_node:	Name hashlist node
1615 1616 1617 1618 1619 1620 1621 1622
 *	@ifalias:	SNMP alias
 *	@mem_end:	Shared memory end
 *	@mem_start:	Shared memory start
 *	@base_addr:	Device I/O address
 *	@irq:		Device IRQ number
 *
 *	@state:		Generic network queuing layer state, see netdev_state_t
 *	@dev_list:	The global list of network devices
B
Bjorn Helgaas 已提交
1623 1624 1625 1626
 *	@napi_list:	List entry used for polling NAPI devices
 *	@unreg_list:	List entry  when we are unregistering the
 *			device; see the function unregister_netdev
 *	@close_list:	List entry used when we are closing the device
1627 1628
 *	@ptype_all:     Device-specific packet handlers for all protocols
 *	@ptype_specific: Device-specific, protocol-specific packet handlers
1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
 *
 *	@adj_list:	Directly linked devices, like slaves for bonding
 *	@features:	Currently active device features
 *	@hw_features:	User-changeable features
 *
 *	@wanted_features:	User-requested features
 *	@vlan_features:		Mask of features inheritable by VLAN devices
 *
 *	@hw_enc_features:	Mask of features inherited by encapsulating devices
 *				This field indicates what encapsulation
 *				offloads the hardware is capable of doing,
 *				and drivers will need to set them appropriately.
 *
 *	@mpls_features:	Mask of features inheritable by MPLS
1643
 *	@gso_partial_features: value(s) from NETIF_F_GSO\*
1644 1645
 *
 *	@ifindex:	interface index
B
Bjorn Helgaas 已提交
1646
 *	@group:		The group the device belongs to
1647 1648 1649 1650 1651 1652 1653 1654
 *
 *	@stats:		Statistics struct, which was left as a legacy, use
 *			rtnl_link_stats64 instead
 *
 *	@rx_dropped:	Dropped packets by core network,
 *			do not use this in drivers
 *	@tx_dropped:	Dropped packets by core network,
 *			do not use this in drivers
1655 1656
 *	@rx_nohandler:	nohandler dropped packets by core network on
 *			inactive devices, do not use this in drivers
1657 1658
 *	@carrier_up_count:	Number of times the carrier has been up
 *	@carrier_down_count:	Number of times the carrier has been down
1659 1660 1661 1662 1663 1664 1665 1666 1667
 *
 *	@wireless_handlers:	List of functions to handle Wireless Extensions,
 *				instead of ioctl,
 *				see <net/iw_handler.h> for details.
 *	@wireless_data:	Instance data managed by the core of wireless extensions
 *
 *	@netdev_ops:	Includes several pointers to callbacks,
 *			if one wants to override the ndo_*() functions
 *	@ethtool_ops:	Management operations
1668
 *	@l3mdev_ops:	Layer 3 master device operations
1669 1670
 *	@ndisc_ops:	Includes callbacks for different IPv6 neighbour
 *			discovery handling. Necessary for e.g. 6LoWPAN.
1671 1672
 *	@xfrmdev_ops:	Transformation offload operations
 *	@tlsdev_ops:	Transport Layer Security offload operations
E
Eric W. Biederman 已提交
1673
 *	@header_ops:	Includes callbacks for creating,parsing,caching,etc
1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
 *			of Layer 2 headers.
 *
 *	@flags:		Interface flags (a la BSD)
 *	@priv_flags:	Like 'flags' but invisible to userspace,
 *			see if.h for the definitions
 *	@gflags:	Global flags ( kept as legacy )
 *	@padded:	How much padding added by alloc_netdev()
 *	@operstate:	RFC2863 operstate
 *	@link_mode:	Mapping policy to operstate
 *	@if_port:	Selectable AUI, TP, ...
 *	@dma:		DMA channel
 *	@mtu:		Interface MTU value
1686 1687
 *	@min_mtu:	Interface Minimum MTU value
 *	@max_mtu:	Interface Maximum MTU value
1688
 *	@type:		Interface hardware type
1689
 *	@hard_header_len: Maximum hardware header length.
1690
 *	@min_header_len:  Minimum hardware header length
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
 *
 *	@needed_headroom: Extra headroom the hardware may need, but not in all
 *			  cases can this be guaranteed
 *	@needed_tailroom: Extra tailroom the hardware may need, but not in all
 *			  cases can this be guaranteed. Some cases also use
 *			  LL_MAX_HEADER instead to allocate the skb
 *
 *	interface address info:
 *
 * 	@perm_addr:		Permanent hw address
 * 	@addr_assign_type:	Hw address assignment type
 * 	@addr_len:		Hardware address length
T
Taehee Yoo 已提交
1703 1704
 *	@upper_level:		Maximum depth level of upper devices.
 *	@lower_level:		Maximum depth level of lower devices.
1705
 *	@neigh_priv_len:	Used in neigh_alloc()
1706 1707 1708 1709 1710
 * 	@dev_id:		Used to differentiate devices that share
 * 				the same link layer address
 * 	@dev_port:		Used to differentiate devices that share
 * 				the same function
 *	@addr_list_lock:	XXX: need comments on this one
1711
 *	@name_assign_type:	network interface name assignment type
B
Bjorn Helgaas 已提交
1712
 *	@uc_promisc:		Counter that indicates promiscuous mode
1713 1714 1715
 *				has been enabled due to the need to listen to
 *				additional unicast addresses in a device that
 *				does not implement ndo_set_rx_mode()
1716 1717 1718 1719
 *	@uc:			unicast mac addresses
 *	@mc:			multicast mac addresses
 *	@dev_addrs:		list of device hw addresses
 *	@queues_kset:		Group of all Kobjects in the Tx and RX queues
B
Bjorn Helgaas 已提交
1720 1721 1722
 *	@promiscuity:		Number of times the NIC is told to work in
 *				promiscuous mode; if it becomes 0 the NIC will
 *				exit promiscuous mode
1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
 *	@allmulti:		Counter, enables or disables allmulticast mode
 *
 *	@vlan_info:	VLAN info
 *	@dsa_ptr:	dsa specific data
 *	@tipc_ptr:	TIPC specific data
 *	@atalk_ptr:	AppleTalk link
 *	@ip_ptr:	IPv4 specific data
 *	@dn_ptr:	DECnet specific data
 *	@ip6_ptr:	IPv6 specific data
 *	@ax25_ptr:	AX.25 specific data
 *	@ieee80211_ptr:	IEEE 802.11 specific data, assign before registering
1734 1735 1736
 *	@ieee802154_ptr: IEEE 802.15.4 low-rate Wireless Personal Area Network
 *			 device struct
 *	@mpls_ptr:	mpls_dev struct pointer
1737 1738 1739 1740 1741 1742 1743 1744
 *
 *	@dev_addr:	Hw address (before bcast,
 *			because most packets are unicast)
 *
 *	@_rx:			Array of RX queues
 *	@num_rx_queues:		Number of RX queues
 *				allocated at register_netdev() time
 *	@real_num_rx_queues: 	Number of RX queues currently active in device
1745 1746
 *	@xdp_prog:		XDP sockets filter program pointer
 *	@gro_flush_timeout:	timeout for GRO layer in NAPI
1747 1748
 *	@napi_defer_hard_irqs:	If not zero, provides a counter that would
 *				allow to avoid NIC hard IRQ, on busy queues.
1749 1750 1751
 *
 *	@rx_handler:		handler for received packets
 *	@rx_handler_data: 	XXX: need comments on this one
1752 1753
 *	@miniq_ingress:		ingress/clsact qdisc specific data for
 *				ingress processing
1754
 *	@ingress_queue:		XXX: need comments on this one
L
Lukas Wunner 已提交
1755
 *	@nf_hooks_ingress:	netfilter hooks executed for ingress packets
1756 1757
 *	@broadcast:		hw bcast address
 *
1758 1759 1760 1761 1762 1763
 *	@rx_cpu_rmap:	CPU reverse-mapping for RX completion interrupts,
 *			indexed by RX queue number. Assigned by driver.
 *			This must only be set if the ndo_rx_flow_steer
 *			operation is defined
 *	@index_hlist:		Device index hash chain
 *
1764 1765 1766 1767 1768 1769
 *	@_tx:			Array of TX queues
 *	@num_tx_queues:		Number of TX queues allocated at alloc_netdev_mq() time
 *	@real_num_tx_queues: 	Number of TX queues currently active in device
 *	@qdisc:			Root qdisc from userspace point of view
 *	@tx_queue_len:		Max frames per queue allowed
 *	@tx_global_lock: 	XXX: need comments on this one
1770 1771 1772
 *	@xdp_bulkq:		XDP device bulk queue
 *	@xps_cpus_map:		all CPUs map for XPS device
 *	@xps_rxqs_map:		all RXQs map for XPS device
1773 1774
 *
 *	@xps_maps:	XXX: need comments on this one
1775 1776
 *	@miniq_egress:		clsact qdisc specific data for
 *				egress processing
1777
 *	@qdisc_hash:		qdisc hash table
1778
 *	@watchdog_timeo:	Represents the timeout that is used by
B
Bjorn Helgaas 已提交
1779
 *				the watchdog (see dev_watchdog())
1780 1781
 *	@watchdog_timer:	List of timers
 *
1782
 *	@proto_down_reason:	reason a netdev interface is held down
1783 1784 1785 1786 1787 1788 1789 1790 1791
 *	@pcpu_refcnt:		Number of references to this device
 *	@todo_list:		Delayed register/unregister
 *	@link_watch_list:	XXX: need comments on this one
 *
 *	@reg_state:		Register/unregister state machine
 *	@dismantle:		Device is going to be freed
 *	@rtnl_link_state:	This enum represents the phases of creating
 *				a new link
 *
1792 1793
 *	@needs_free_netdev:	Should unregister perform free_netdev?
 *	@priv_destructor:	Called from unregister
1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
 *	@npinfo:		XXX: need comments on this one
 * 	@nd_net:		Network namespace this network device is inside
 *
 * 	@ml_priv:	Mid-layer private
 * 	@lstats:	Loopback statistics
 * 	@tstats:	Tunnel statistics
 * 	@dstats:	Dummy statistics
 * 	@vstats:	Virtual ethernet statistics
 *
 *	@garp_port:	GARP
 *	@mrp_port:	MRP
 *
 *	@dev:		Class/net/name entry
 *	@sysfs_groups:	Space for optional device, statistics and wireless
 *			sysfs groups
 *
 *	@sysfs_rx_queue_group:	Space for optional per-rx queue attributes
 *	@rtnl_link_ops:	Rtnl_link_ops
 *
 *	@gso_max_size:	Maximum size of generic segmentation offload
 *	@gso_max_segs:	Maximum number of segments that can be passed to the
 *			NIC for GSO
 *
 *	@dcbnl_ops:	Data Center Bridging netlink ops
 *	@num_tc:	Number of traffic classes in the net device
 *	@tc_to_txq:	XXX: need comments on this one
R
Randy Dunlap 已提交
1820
 *	@prio_tc_map:	XXX: need comments on this one
1821 1822 1823 1824 1825 1826
 *
 *	@fcoe_ddp_xid:	Max exchange id for FCoE LRO by ddp
 *
 *	@priomap:	XXX: need comments on this one
 *	@phydev:	Physical device may attach itself
 *			for hardware timestamping
1827
 *	@sfp_bus:	attached &struct sfp_bus structure.
1828 1829 1830
 *
 *	@qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
 *	@qdisc_running_key: lockdep class annotating Qdisc->running seqcount
1831
 *
1832 1833 1834 1835
 *	@proto_down:	protocol port state information can be sent to the
 *			switch driver and used to set the phys state of the
 *			switch port.
 *
1836 1837
 *	@wol_enabled:	Wake-on-LAN is enabled
 *
1838 1839 1840 1841
 *	@net_notifier_list:	List of per-net netdev notifier block
 *				that follow this device when it is moved
 *				to another network namespace.
 *
1842 1843
 *	@macsec_ops:    MACsec offloading ops
 *
1844 1845 1846
 *	@udp_tunnel_nic_info:	static structure describing the UDP tunnel
 *				offload capabilities of the device
 *	@udp_tunnel_nic:	UDP tunnel offload state
1847
 *	@xdp_state:		stores info on attached XDP BPF programs
1848
 *
1849 1850 1851 1852 1853
 *	@nested_level:	Used as as a parameter of spin_lock_nested() of
 *			dev->addr_list_lock.
 *	@unlink_list:	As netif_addr_lock() can be called recursively,
 *			keep a list of interfaces to be deleted.
 *
L
Linus Torvalds 已提交
1854 1855 1856 1857
 *	FIXME: cleanup struct net_device such that network protocol info
 *	moves out.
 */

E
Eric Dumazet 已提交
1858
struct net_device {
L
Linus Torvalds 已提交
1859
	char			name[IFNAMSIZ];
1860
	struct netdev_name_node	*name_node;
1861
	struct dev_ifalias	__rcu *ifalias;
L
Linus Torvalds 已提交
1862 1863 1864 1865
	/*
	 *	I/O specific fields
	 *	FIXME: Merge these and struct ifmap into one
	 */
1866 1867 1868 1869
	unsigned long		mem_end;
	unsigned long		mem_start;
	unsigned long		base_addr;
	int			irq;
L
Linus Torvalds 已提交
1870 1871

	/*
1872 1873
	 *	Some hardware also needs these fields (state,dev_list,
	 *	napi_list,unreg_list,close_list) but they are not
L
Linus Torvalds 已提交
1874 1875 1876 1877 1878
	 *	part of the usual set specified in Space.c.
	 */

	unsigned long		state;

1879
	struct list_head	dev_list;
1880
	struct list_head	napi_list;
1881
	struct list_head	unreg_list;
1882
	struct list_head	close_list;
1883 1884
	struct list_head	ptype_all;
	struct list_head	ptype_specific;
1885 1886 1887 1888 1889 1890

	struct {
		struct list_head upper;
		struct list_head lower;
	} adj_list;

1891 1892 1893 1894
	netdev_features_t	features;
	netdev_features_t	hw_features;
	netdev_features_t	wanted_features;
	netdev_features_t	vlan_features;
1895
	netdev_features_t	hw_enc_features;
S
Simon Horman 已提交
1896
	netdev_features_t	mpls_features;
1897
	netdev_features_t	gso_partial_features;
1898

L
Linus Torvalds 已提交
1899
	int			ifindex;
1900
	int			group;
L
Linus Torvalds 已提交
1901

R
Rusty Russell 已提交
1902
	struct net_device_stats	stats;
1903 1904 1905

	atomic_long_t		rx_dropped;
	atomic_long_t		tx_dropped;
1906
	atomic_long_t		rx_nohandler;
L
Linus Torvalds 已提交
1907

1908 1909 1910 1911
	/* Stats to monitor link on/off, flapping */
	atomic_t		carrier_up_count;
	atomic_t		carrier_down_count;

1912
#ifdef CONFIG_WIRELESS_EXT
B
Bjorn Helgaas 已提交
1913 1914
	const struct iw_handler_def *wireless_handlers;
	struct iw_public_data	*wireless_data;
1915
#endif
1916
	const struct net_device_ops *netdev_ops;
1917
	const struct ethtool_ops *ethtool_ops;
1918 1919 1920
#ifdef CONFIG_NET_L3_MASTER_DEV
	const struct l3mdev_ops	*l3mdev_ops;
#endif
1921 1922 1923
#if IS_ENABLED(CONFIG_IPV6)
	const struct ndisc_ops *ndisc_ops;
#endif
L
Linus Torvalds 已提交
1924

1925
#ifdef CONFIG_XFRM_OFFLOAD
1926 1927 1928
	const struct xfrmdev_ops *xfrmdev_ops;
#endif

I
Ilya Lesokhin 已提交
1929 1930 1931 1932
#if IS_ENABLED(CONFIG_TLS_DEVICE)
	const struct tlsdev_ops *tlsdev_ops;
#endif

1933 1934
	const struct header_ops *header_ops;

1935 1936 1937
	unsigned int		flags;
	unsigned int		priv_flags;

L
Linus Torvalds 已提交
1938
	unsigned short		gflags;
1939
	unsigned short		padded;
L
Linus Torvalds 已提交
1940

1941 1942
	unsigned char		operstate;
	unsigned char		link_mode;
S
Stefan Rompf 已提交
1943

1944 1945
	unsigned char		if_port;
	unsigned char		dma;
1946

1947 1948 1949 1950 1951
	/* Note : dev->mtu is often read without holding a lock.
	 * Writers usually hold RTNL.
	 * It is recommended to use READ_ONCE() to annotate the reads,
	 * and to use WRITE_ONCE() to annotate the writes.
	 */
1952
	unsigned int		mtu;
1953 1954
	unsigned int		min_mtu;
	unsigned int		max_mtu;
1955 1956
	unsigned short		type;
	unsigned short		hard_header_len;
1957
	unsigned char		min_header_len;
1958
	unsigned char		name_assign_type;
L
Linus Torvalds 已提交
1959

1960 1961 1962
	unsigned short		needed_headroom;
	unsigned short		needed_tailroom;

L
Linus Torvalds 已提交
1963
	/* Interface address info. */
1964 1965 1966
	unsigned char		perm_addr[MAX_ADDR_LEN];
	unsigned char		addr_assign_type;
	unsigned char		addr_len;
T
Taehee Yoo 已提交
1967 1968
	unsigned char		upper_level;
	unsigned char		lower_level;
1969

1970
	unsigned short		neigh_priv_len;
1971 1972
	unsigned short          dev_id;
	unsigned short          dev_port;
J
Jiri Pirko 已提交
1973
	spinlock_t		addr_list_lock;
1974

1975 1976 1977 1978
	struct netdev_hw_addr_list	uc;
	struct netdev_hw_addr_list	mc;
	struct netdev_hw_addr_list	dev_addrs;

1979 1980
#ifdef CONFIG_SYSFS
	struct kset		*queues_kset;
1981 1982 1983
#endif
#ifdef CONFIG_LOCKDEP
	struct list_head	unlink_list;
1984
#endif
1985 1986
	unsigned int		promiscuity;
	unsigned int		allmulti;
1987 1988 1989 1990
	bool			uc_promisc;
#ifdef CONFIG_LOCKDEP
	unsigned char		nested_level;
#endif
L
Linus Torvalds 已提交
1991 1992


B
Bjorn Helgaas 已提交
1993
	/* Protocol-specific pointers */
1994

1995
#if IS_ENABLED(CONFIG_VLAN_8021Q)
1996
	struct vlan_info __rcu	*vlan_info;
1997
#endif
1998
#if IS_ENABLED(CONFIG_NET_DSA)
1999
	struct dsa_port		*dsa_ptr;
2000 2001
#endif
#if IS_ENABLED(CONFIG_TIPC)
2002
	struct tipc_bearer __rcu *tipc_ptr;
2003
#endif
2004
#if IS_ENABLED(CONFIG_IRDA) || IS_ENABLED(CONFIG_ATALK)
2005
	void 			*atalk_ptr;
2006
#endif
2007
	struct in_device __rcu	*ip_ptr;
2008
#if IS_ENABLED(CONFIG_DECNET)
2009
	struct dn_dev __rcu     *dn_ptr;
2010
#endif
2011
	struct inet6_dev __rcu	*ip6_ptr;
2012
#if IS_ENABLED(CONFIG_AX25)
2013
	void			*ax25_ptr;
2014
#endif
2015
	struct wireless_dev	*ieee80211_ptr;
2016
	struct wpan_dev		*ieee802154_ptr;
R
Robert Shearman 已提交
2017 2018 2019
#if IS_ENABLED(CONFIG_MPLS_ROUTING)
	struct mpls_dev __rcu	*mpls_ptr;
#endif
L
Linus Torvalds 已提交
2020

2021
/*
E
Eric Dumazet 已提交
2022
 * Cache lines mostly used on receive path (including eth_type_trans())
2023 2024
 */
	/* Interface address info used in eth_type_trans() */
2025
	unsigned char		*dev_addr;
2026

T
Tom Herbert 已提交
2027 2028
	struct netdev_rx_queue	*_rx;
	unsigned int		num_rx_queues;
2029
	unsigned int		real_num_rx_queues;
T
Tom Herbert 已提交
2030

2031
	struct bpf_prog __rcu	*xdp_prog;
2032
	unsigned long		gro_flush_timeout;
2033
	int			napi_defer_hard_irqs;
2034 2035
	rx_handler_func_t __rcu	*rx_handler;
	void __rcu		*rx_handler_data;
2036

2037
#ifdef CONFIG_NET_CLS_ACT
2038
	struct mini_Qdisc __rcu	*miniq_ingress;
2039
#endif
2040
	struct netdev_queue __rcu *ingress_queue;
2041
#ifdef CONFIG_NETFILTER_INGRESS
2042
	struct nf_hook_entries __rcu *nf_hooks_ingress;
2043
#endif
2044

2045
	unsigned char		broadcast[MAX_ADDR_LEN];
2046 2047 2048 2049
#ifdef CONFIG_RFS_ACCEL
	struct cpu_rmap		*rx_cpu_rmap;
#endif
	struct hlist_node	index_hlist;
E
Eric Dumazet 已提交
2050 2051 2052 2053

/*
 * Cache lines mostly used on transmit path
 */
2054 2055
	struct netdev_queue	*_tx ____cacheline_aligned_in_smp;
	unsigned int		num_tx_queues;
2056
	unsigned int		real_num_tx_queues;
2057
	struct Qdisc		*qdisc;
2058
	unsigned int		tx_queue_len;
2059
	spinlock_t		tx_global_lock;
2060 2061

	struct xdp_dev_bulk_queue __percpu *xdp_bulkq;
E
Eric Dumazet 已提交
2062

T
Tom Herbert 已提交
2063
#ifdef CONFIG_XPS
2064 2065
	struct xps_dev_maps __rcu *xps_cpus_map;
	struct xps_dev_maps __rcu *xps_rxqs_map;
T
Tom Herbert 已提交
2066
#endif
D
Daniel Borkmann 已提交
2067
#ifdef CONFIG_NET_CLS_ACT
2068
	struct mini_Qdisc __rcu	*miniq_egress;
D
Daniel Borkmann 已提交
2069
#endif
2070

2071 2072 2073
#ifdef CONFIG_NET_SCHED
	DECLARE_HASHTABLE	(qdisc_hash, 4);
#endif
2074 2075
	/* These may be needed for future network-power-down code. */
	struct timer_list	watchdog_timer;
2076
	int			watchdog_timeo;
2077

2078 2079
	u32                     proto_down_reason;

L
Linus Torvalds 已提交
2080
	struct list_head	todo_list;
2081
	int __percpu		*pcpu_refcnt;
L
Linus Torvalds 已提交
2082

2083
	struct list_head	link_watch_list;
2084

L
Linus Torvalds 已提交
2085
	enum { NETREG_UNINITIALIZED=0,
2086
	       NETREG_REGISTERED,	/* completed register_netdevice */
L
Linus Torvalds 已提交
2087 2088 2089
	       NETREG_UNREGISTERING,	/* called unregister_netdevice */
	       NETREG_UNREGISTERED,	/* completed unregister todo */
	       NETREG_RELEASED,		/* called free_netdev */
2090
	       NETREG_DUMMY,		/* dummy device for NAPI poll */
2091 2092
	} reg_state:8;

2093
	bool dismantle;
2094 2095 2096 2097 2098

	enum {
		RTNL_LINK_INITIALIZED,
		RTNL_LINK_INITIALIZING,
	} rtnl_link_state:16;
L
Linus Torvalds 已提交
2099

2100 2101
	bool needs_free_netdev;
	void (*priv_destructor)(struct net_device *dev);
L
Linus Torvalds 已提交
2102 2103

#ifdef CONFIG_NETPOLL
2104
	struct netpoll_info __rcu	*npinfo;
L
Linus Torvalds 已提交
2105
#endif
2106

2107
	possible_net_t			nd_net;
2108

D
David S. Miller 已提交
2109
	/* mid-layer private */
E
Eric Dumazet 已提交
2110
	union {
2111 2112
		void					*ml_priv;
		struct pcpu_lstats __percpu		*lstats;
2113
		struct pcpu_sw_netstats __percpu	*tstats;
2114
		struct pcpu_dstats __percpu		*dstats;
E
Eric Dumazet 已提交
2115
	};
2116

2117
#if IS_ENABLED(CONFIG_GARP)
E
Eric Dumazet 已提交
2118
	struct garp_port __rcu	*garp_port;
2119 2120
#endif
#if IS_ENABLED(CONFIG_MRP)
2121
	struct mrp_port __rcu	*mrp_port;
2122
#endif
L
Linus Torvalds 已提交
2123

B
Bjorn Helgaas 已提交
2124
	struct device		dev;
2125
	const struct attribute_group *sysfs_groups[4];
2126
	const struct attribute_group *sysfs_rx_queue_group;
P
Patrick McHardy 已提交
2127 2128

	const struct rtnl_link_ops *rtnl_link_ops;
2129

2130 2131 2132
	/* for setting kernel sock attribute on TCP connection setup */
#define GSO_MAX_SIZE		65536
	unsigned int		gso_max_size;
2133 2134
#define GSO_MAX_SEGS		65535
	u16			gso_max_segs;
2135

J
Jeff Kirsher 已提交
2136
#ifdef CONFIG_DCB
2137
	const struct dcbnl_rtnl_ops *dcbnl_ops;
2138
#endif
2139
	s16			num_tc;
B
Bjorn Helgaas 已提交
2140 2141
	struct netdev_tc_txq	tc_to_txq[TC_MAX_QUEUE];
	u8			prio_tc_map[TC_BITMASK + 1];
2142

2143
#if IS_ENABLED(CONFIG_FCOE)
2144
	unsigned int		fcoe_ddp_xid;
2145
#endif
2146
#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2147
	struct netprio_map __rcu *priomap;
2148
#endif
B
Bjorn Helgaas 已提交
2149
	struct phy_device	*phydev;
2150
	struct sfp_bus		*sfp_bus;
2151 2152
	struct lock_class_key	*qdisc_tx_busylock;
	struct lock_class_key	*qdisc_running_key;
B
Bjorn Helgaas 已提交
2153
	bool			proto_down;
2154
	unsigned		wol_enabled:1;
2155 2156

	struct list_head	net_notifier_list;
2157 2158 2159 2160 2161

#if IS_ENABLED(CONFIG_MACSEC)
	/* MACsec management functions */
	const struct macsec_ops *macsec_ops;
#endif
2162 2163
	const struct udp_tunnel_nic_info	*udp_tunnel_nic_info;
	struct udp_tunnel_nic	*udp_tunnel_nic;
2164 2165 2166

	/* protected by rtnl_lock */
	struct bpf_xdp_entity	xdp_state[__MAX_XDP_MODE];
L
Linus Torvalds 已提交
2167
};
2168
#define to_net_dev(d) container_of(d, struct net_device, dev)
L
Linus Torvalds 已提交
2169

D
David S. Miller 已提交
2170 2171 2172 2173 2174 2175 2176
static inline bool netif_elide_gro(const struct net_device *dev)
{
	if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
		return true;
	return false;
}

L
Linus Torvalds 已提交
2177 2178
#define	NETDEV_ALIGN		32

2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194
static inline
int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
{
	return dev->prio_tc_map[prio & TC_BITMASK];
}

static inline
int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
{
	if (tc >= dev->num_tc)
		return -EINVAL;

	dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
	return 0;
}

2195
int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
2196 2197 2198
void netdev_reset_tc(struct net_device *dev);
int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
2199 2200 2201 2202 2203 2204 2205

static inline
int netdev_get_num_tc(struct net_device *dev)
{
	return dev->num_tc;
}

2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221
static inline void net_prefetch(void *p)
{
	prefetch(p);
#if L1_CACHE_BYTES < 128
	prefetch((u8 *)p + L1_CACHE_BYTES);
#endif
}

static inline void net_prefetchw(void *p)
{
	prefetchw(p);
#if L1_CACHE_BYTES < 128
	prefetchw((u8 *)p + L1_CACHE_BYTES);
#endif
}

2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232
void netdev_unbind_sb_channel(struct net_device *dev,
			      struct net_device *sb_dev);
int netdev_bind_sb_channel_queue(struct net_device *dev,
				 struct net_device *sb_dev,
				 u8 tc, u16 count, u16 offset);
int netdev_set_sb_channel(struct net_device *dev, u16 channel);
static inline int netdev_get_sb_channel(struct net_device *dev)
{
	return max_t(int, -dev->num_tc, 0);
}

2233 2234 2235 2236 2237 2238 2239
static inline
struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
					 unsigned int index)
{
	return &dev->_tx[index];
}

2240 2241 2242 2243 2244 2245
static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
						    const struct sk_buff *skb)
{
	return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
}

2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257
static inline void netdev_for_each_tx_queue(struct net_device *dev,
					    void (*f)(struct net_device *,
						      struct netdev_queue *,
						      void *),
					    void *arg)
{
	unsigned int i;

	for (i = 0; i < dev->num_tx_queues; i++)
		f(dev, &dev->_tx[i], arg);
}

2258 2259 2260 2261 2262
#define netdev_lockdep_set_classes(dev)				\
{								\
	static struct lock_class_key qdisc_tx_busylock_key;	\
	static struct lock_class_key qdisc_running_key;		\
	static struct lock_class_key qdisc_xmit_lock_key;	\
2263
	static struct lock_class_key dev_addr_list_lock_key;	\
2264 2265 2266 2267
	unsigned int i;						\
								\
	(dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key;	\
	(dev)->qdisc_running_key = &qdisc_running_key;		\
2268 2269
	lockdep_set_class(&(dev)->addr_list_lock,		\
			  &dev_addr_list_lock_key);		\
2270 2271 2272 2273 2274
	for (i = 0; i < (dev)->num_tx_queues; i++)		\
		lockdep_set_class(&(dev)->_tx[i]._xmit_lock,	\
				  &qdisc_xmit_lock_key);	\
}

2275 2276
u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
		     struct net_device *sb_dev);
2277 2278 2279
struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
					 struct sk_buff *skb,
					 struct net_device *sb_dev);
2280

2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300
/* returns the headroom that the master device needs to take in account
 * when forwarding to this dev
 */
static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
{
	return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
}

static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
{
	if (dev->netdev_ops->ndo_set_rx_headroom)
		dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
}

/* set the device rx headroom to the dev's default */
static inline void netdev_reset_rx_headroom(struct net_device *dev)
{
	netdev_set_rx_headroom(dev, -1);
}

2301 2302 2303 2304 2305 2306
/*
 * Net namespace inlines
 */
static inline
struct net *dev_net(const struct net_device *dev)
{
E
Eric Dumazet 已提交
2307
	return read_pnet(&dev->nd_net);
2308 2309 2310
}

static inline
2311
void dev_net_set(struct net_device *dev, struct net *net)
2312
{
2313
	write_pnet(&dev->nd_net, net);
2314 2315
}

2316 2317 2318 2319 2320 2321
/**
 *	netdev_priv - access network device private data
 *	@dev: network device
 *
 * Get network device private data
 */
2322
static inline void *netdev_priv(const struct net_device *dev)
L
Linus Torvalds 已提交
2323
{
2324
	return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
L
Linus Torvalds 已提交
2325 2326 2327 2328 2329
}

/* Set the sysfs physical device reference for the network logical device
 * if set prior to registration will cause a symlink during initialization.
 */
2330
#define SET_NETDEV_DEV(net, pdev)	((net)->dev.parent = (pdev))
L
Linus Torvalds 已提交
2331

2332
/* Set the sysfs device type for the network logical device to allow
2333
 * fine-grained identification of different network device types. For
B
Bjorn Helgaas 已提交
2334
 * example Ethernet, Wireless LAN, Bluetooth, WiMAX etc.
2335 2336 2337
 */
#define SET_NETDEV_DEVTYPE(net, devtype)	((net)->dev.type = (devtype))

E
Eric Dumazet 已提交
2338 2339 2340 2341 2342
/* Default NAPI poll() weight
 * Device drivers are strongly advised to not use bigger value
 */
#define NAPI_POLL_WEIGHT 64

2343
/**
B
Bjorn Helgaas 已提交
2344
 *	netif_napi_add - initialize a NAPI context
2345
 *	@dev:  network device
B
Bjorn Helgaas 已提交
2346
 *	@napi: NAPI context
2347 2348 2349
 *	@poll: polling function
 *	@weight: default weight
 *
B
Bjorn Helgaas 已提交
2350 2351
 * netif_napi_add() must be used to initialize a NAPI context prior to calling
 * *any* of the other NAPI-related functions.
2352
 */
2353 2354
void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
		    int (*poll)(struct napi_struct *, int), int weight);
2355

E
Eric Dumazet 已提交
2356
/**
B
Bjorn Helgaas 已提交
2357
 *	netif_tx_napi_add - initialize a NAPI context
E
Eric Dumazet 已提交
2358
 *	@dev:  network device
B
Bjorn Helgaas 已提交
2359
 *	@napi: NAPI context
E
Eric Dumazet 已提交
2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
 *	@poll: polling function
 *	@weight: default weight
 *
 * This variant of netif_napi_add() should be used from drivers using NAPI
 * to exclusively poll a TX queue.
 * This will avoid we add it into napi_hash[], thus polluting this hash table.
 */
static inline void netif_tx_napi_add(struct net_device *dev,
				     struct napi_struct *napi,
				     int (*poll)(struct napi_struct *, int),
				     int weight)
{
	set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
	netif_napi_add(dev, napi, poll, weight);
}

2376 2377 2378 2379 2380 2381 2382 2383 2384 2385
/**
 *  __netif_napi_del - remove a NAPI context
 *  @napi: NAPI context
 *
 * Warning: caller must observe RCU grace period before freeing memory
 * containing @napi. Drivers might want to call this helper to combine
 * all the needed RCU grace periods into a single one.
 */
void __netif_napi_del(struct napi_struct *napi);

2386
/**
B
Bjorn Helgaas 已提交
2387 2388
 *  netif_napi_del - remove a NAPI context
 *  @napi: NAPI context
2389
 *
B
Bjorn Helgaas 已提交
2390
 *  netif_napi_del() removes a NAPI context from the network device NAPI list
2391
 */
2392 2393 2394 2395 2396
static inline void netif_napi_del(struct napi_struct *napi)
{
	__netif_napi_del(napi);
	synchronize_net();
}
2397 2398

struct napi_gro_cb {
2399
	/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
B
Bjorn Helgaas 已提交
2400
	void	*frag0;
2401

2402 2403 2404
	/* Length of frag0. */
	unsigned int frag0_len;

2405
	/* This indicates where we are processing relative to skb->data. */
B
Bjorn Helgaas 已提交
2406
	int	data_offset;
2407

2408
	/* This is non-zero if the packet cannot be merged with the new skb. */
2409 2410 2411 2412
	u16	flush;

	/* Save the IP ID here and check when we get to the transport layer */
	u16	flush_id;
2413 2414

	/* Number of segments aggregated. */
2415 2416
	u16	count;

2417 2418 2419
	/* Start offset for remote checksum offload */
	u16	gro_remcsum_start;

2420 2421
	/* jiffies when first packet was created/queued */
	unsigned long age;
2422

T
Tom Herbert 已提交
2423
	/* Used in ipv6_gro_receive() and foo-over-udp */
2424 2425
	u16	proto;

2426 2427 2428
	/* This is non-zero if the packet may be of the same flow. */
	u8	same_flow:1;

2429 2430
	/* Used in tunnel GRO receive */
	u8	encap_mark:1;
T
Tom Herbert 已提交
2431 2432 2433 2434

	/* GRO checksum is valid */
	u8	csum_valid:1;

2435 2436
	/* Number of checksums via CHECKSUM_UNNECESSARY */
	u8	csum_cnt:3;
2437

2438 2439 2440 2441 2442
	/* Free the skb? */
	u8	free:2;
#define NAPI_GRO_FREE		  1
#define NAPI_GRO_FREE_STOLEN_HEAD 2

2443 2444 2445
	/* Used in foo-over-udp, set in udp[46]_gro_receive */
	u8	is_ipv6:1;

2446 2447 2448
	/* Used in GRE, set in fou/gue_gro_receive */
	u8	is_fou:1;

2449 2450 2451
	/* Used to determine if flush_id can be ignored */
	u8	is_atomic:1;

S
Sabrina Dubroca 已提交
2452 2453 2454
	/* Number of gro_receive callbacks this packet already went through */
	u8 recursion_counter:4;

2455 2456
	/* GRO is done by frag_list pointer chaining. */
	u8	is_flist:1;
2457

2458 2459 2460
	/* used to support CHECKSUM_COMPLETE for tunneling protocols */
	__wsum	csum;

2461 2462
	/* used in skb_gro_receive() slow path */
	struct sk_buff *last;
2463 2464 2465
};

#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
2466

S
Sabrina Dubroca 已提交
2467 2468 2469 2470 2471 2472
#define GRO_RECURSION_LIMIT 15
static inline int gro_recursion_inc_test(struct sk_buff *skb)
{
	return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
}

2473 2474 2475 2476
typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
					       struct list_head *head,
					       struct sk_buff *skb)
S
Sabrina Dubroca 已提交
2477 2478 2479 2480 2481 2482 2483 2484 2485
{
	if (unlikely(gro_recursion_inc_test(skb))) {
		NAPI_GRO_CB(skb)->flush |= 1;
		return NULL;
	}

	return cb(head, skb);
}

2486 2487 2488 2489 2490 2491
typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
					    struct sk_buff *);
static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
						  struct sock *sk,
						  struct list_head *head,
						  struct sk_buff *skb)
S
Sabrina Dubroca 已提交
2492 2493 2494 2495 2496 2497 2498 2499 2500
{
	if (unlikely(gro_recursion_inc_test(skb))) {
		NAPI_GRO_CB(skb)->flush |= 1;
		return NULL;
	}

	return cb(sk, head, skb);
}

L
Linus Torvalds 已提交
2501
struct packet_type {
D
David S. Miller 已提交
2502
	__be16			type;	/* This is really htons(ether_type). */
2503
	bool			ignore_outgoing;
D
David S. Miller 已提交
2504 2505 2506 2507 2508
	struct net_device	*dev;	/* NULL is wildcarded here	     */
	int			(*func) (struct sk_buff *,
					 struct net_device *,
					 struct packet_type *,
					 struct net_device *);
2509 2510 2511
	void			(*list_func) (struct list_head *,
					      struct packet_type *,
					      struct net_device *);
2512 2513
	bool			(*id_match)(struct packet_type *ptype,
					    struct sock *sk);
L
Linus Torvalds 已提交
2514 2515 2516 2517
	void			*af_packet_priv;
	struct list_head	list;
};

2518
struct offload_callbacks {
2519
	struct sk_buff		*(*gso_segment)(struct sk_buff *skb,
2520
						netdev_features_t features);
2521 2522
	struct sk_buff		*(*gro_receive)(struct list_head *head,
						struct sk_buff *skb);
2523
	int			(*gro_complete)(struct sk_buff *skb, int nhoff);
2524 2525 2526 2527
};

struct packet_offload {
	__be16			 type;	/* This is really htons(ether_type). */
2528
	u16			 priority;
2529 2530
	struct offload_callbacks callbacks;
	struct list_head	 list;
L
Linus Torvalds 已提交
2531 2532
};

B
Bjorn Helgaas 已提交
2533
/* often modified stats are per-CPU, other are shared (netdev->stats) */
2534 2535 2536 2537 2538 2539
struct pcpu_sw_netstats {
	u64     rx_packets;
	u64     rx_bytes;
	u64     tx_packets;
	u64     tx_bytes;
	struct u64_stats_sync   syncp;
2540
} __aligned(4 * sizeof(u64));
2541 2542

struct pcpu_lstats {
2543 2544
	u64_stats_t packets;
	u64_stats_t bytes;
2545
	struct u64_stats_sync syncp;
2546
} __aligned(2 * sizeof(u64));
2547

2548 2549
void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes);

2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int len)
{
	struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);

	u64_stats_update_begin(&tstats->syncp);
	tstats->rx_bytes += len;
	tstats->rx_packets++;
	u64_stats_update_end(&tstats->syncp);
}

2560 2561 2562 2563 2564
static inline void dev_lstats_add(struct net_device *dev, unsigned int len)
{
	struct pcpu_lstats *lstats = this_cpu_ptr(dev->lstats);

	u64_stats_update_begin(&lstats->syncp);
2565 2566
	u64_stats_add(&lstats->bytes, len);
	u64_stats_inc(&lstats->packets);
2567 2568 2569
	u64_stats_update_end(&lstats->syncp);
}

2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581
#define __netdev_alloc_pcpu_stats(type, gfp)				\
({									\
	typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
	if (pcpu_stats)	{						\
		int __cpu;						\
		for_each_possible_cpu(__cpu) {				\
			typeof(type) *stat;				\
			stat = per_cpu_ptr(pcpu_stats, __cpu);		\
			u64_stats_init(&stat->syncp);			\
		}							\
	}								\
	pcpu_stats;							\
2582 2583
})

2584
#define netdev_alloc_pcpu_stats(type)					\
2585
	__netdev_alloc_pcpu_stats(type, GFP_KERNEL)
2586

2587 2588 2589 2590 2591 2592 2593 2594 2595
enum netdev_lag_tx_type {
	NETDEV_LAG_TX_TYPE_UNKNOWN,
	NETDEV_LAG_TX_TYPE_RANDOM,
	NETDEV_LAG_TX_TYPE_BROADCAST,
	NETDEV_LAG_TX_TYPE_ROUNDROBIN,
	NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
	NETDEV_LAG_TX_TYPE_HASH,
};

2596 2597 2598 2599 2600 2601 2602 2603 2604 2605
enum netdev_lag_hash {
	NETDEV_LAG_HASH_NONE,
	NETDEV_LAG_HASH_L2,
	NETDEV_LAG_HASH_L34,
	NETDEV_LAG_HASH_L23,
	NETDEV_LAG_HASH_E23,
	NETDEV_LAG_HASH_E34,
	NETDEV_LAG_HASH_UNKNOWN,
};

2606 2607
struct netdev_lag_upper_info {
	enum netdev_lag_tx_type tx_type;
2608
	enum netdev_lag_hash hash_type;
2609 2610
};

2611 2612 2613 2614 2615
struct netdev_lag_lower_state_info {
	u8 link_up : 1,
	   tx_enabled : 1;
};

L
Linus Torvalds 已提交
2616 2617
#include <linux/notifier.h>

2618 2619 2620
/* netdevice notifier chain. Please remember to update netdev_cmd_to_name()
 * and the rtnetlink notification exclusion list in rtnetlink_event() when
 * adding new types.
2621
 */
2622 2623 2624 2625
enum netdev_cmd {
	NETDEV_UP	= 1,	/* For now you can't veto a device up/down */
	NETDEV_DOWN,
	NETDEV_REBOOT,		/* Tell a protocol stack a network interface
2626 2627 2628
				   detected a hardware crash and restarted
				   - we can use this eg to kick tcp sessions
				   once done */
2629 2630 2631 2632
	NETDEV_CHANGE,		/* Notify device state change */
	NETDEV_REGISTER,
	NETDEV_UNREGISTER,
	NETDEV_CHANGEMTU,	/* notify after mtu change happened */
P
Petr Machata 已提交
2633 2634
	NETDEV_CHANGEADDR,	/* notify after the address change */
	NETDEV_PRE_CHANGEADDR,	/* notify before the address change */
2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655
	NETDEV_GOING_DOWN,
	NETDEV_CHANGENAME,
	NETDEV_FEAT_CHANGE,
	NETDEV_BONDING_FAILOVER,
	NETDEV_PRE_UP,
	NETDEV_PRE_TYPE_CHANGE,
	NETDEV_POST_TYPE_CHANGE,
	NETDEV_POST_INIT,
	NETDEV_RELEASE,
	NETDEV_NOTIFY_PEERS,
	NETDEV_JOIN,
	NETDEV_CHANGEUPPER,
	NETDEV_RESEND_IGMP,
	NETDEV_PRECHANGEMTU,	/* notify before mtu change happened */
	NETDEV_CHANGEINFODATA,
	NETDEV_BONDING_INFO,
	NETDEV_PRECHANGEUPPER,
	NETDEV_CHANGELOWERSTATE,
	NETDEV_UDP_TUNNEL_PUSH_INFO,
	NETDEV_UDP_TUNNEL_DROP_INFO,
	NETDEV_CHANGE_TX_QUEUE_LEN,
2656 2657 2658 2659
	NETDEV_CVLAN_FILTER_PUSH_INFO,
	NETDEV_CVLAN_FILTER_DROP_INFO,
	NETDEV_SVLAN_FILTER_PUSH_INFO,
	NETDEV_SVLAN_FILTER_DROP_INFO,
2660 2661
};
const char *netdev_cmd_to_name(enum netdev_cmd cmd);
2662

2663 2664
int register_netdevice_notifier(struct notifier_block *nb);
int unregister_netdevice_notifier(struct notifier_block *nb);
2665 2666 2667
int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb);
int unregister_netdevice_notifier_net(struct net *net,
				      struct notifier_block *nb);
2668 2669 2670 2671 2672 2673
int register_netdevice_notifier_dev_net(struct net_device *dev,
					struct notifier_block *nb,
					struct netdev_net_notifier *nn);
int unregister_netdevice_notifier_dev_net(struct net_device *dev,
					  struct notifier_block *nb,
					  struct netdev_net_notifier *nn);
2674 2675

struct netdev_notifier_info {
2676 2677
	struct net_device	*dev;
	struct netlink_ext_ack	*extack;
2678 2679
};

2680 2681 2682 2683 2684 2685 2686
struct netdev_notifier_info_ext {
	struct netdev_notifier_info info; /* must be first */
	union {
		u32 mtu;
	} ext;
};

2687 2688 2689 2690 2691
struct netdev_notifier_change_info {
	struct netdev_notifier_info info; /* must be first */
	unsigned int flags_changed;
};

2692 2693 2694 2695
struct netdev_notifier_changeupper_info {
	struct netdev_notifier_info info; /* must be first */
	struct net_device *upper_dev; /* new upper dev */
	bool master; /* is upper dev master */
B
Bjorn Helgaas 已提交
2696
	bool linking; /* is the notification for link or unlink */
2697
	void *upper_info; /* upper dev info */
2698 2699
};

2700 2701 2702 2703 2704
struct netdev_notifier_changelowerstate_info {
	struct netdev_notifier_info info; /* must be first */
	void *lower_state_info; /* is lower dev state */
};

P
Petr Machata 已提交
2705 2706 2707 2708 2709
struct netdev_notifier_pre_changeaddr_info {
	struct netdev_notifier_info info; /* must be first */
	const unsigned char *dev_addr;
};

2710 2711 2712 2713
static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
					     struct net_device *dev)
{
	info->dev = dev;
2714
	info->extack = NULL;
2715 2716
}

2717 2718 2719 2720 2721 2722
static inline struct net_device *
netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
{
	return info->dev;
}

2723 2724 2725 2726 2727 2728
static inline struct netlink_ext_ack *
netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
{
	return info->extack;
}

2729
int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
2730 2731


L
Linus Torvalds 已提交
2732 2733
extern rwlock_t				dev_base_lock;		/* Device list lock */

2734 2735
#define for_each_netdev(net, d)		\
		list_for_each_entry(d, &(net)->dev_base_head, dev_list)
2736 2737
#define for_each_netdev_reverse(net, d)	\
		list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
2738 2739
#define for_each_netdev_rcu(net, d)		\
		list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
2740 2741 2742 2743
#define for_each_netdev_safe(net, d, n)	\
		list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
#define for_each_netdev_continue(net, d)		\
		list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
2744 2745 2746
#define for_each_netdev_continue_reverse(net, d)		\
		list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \
						     dev_list)
2747 2748
#define for_each_netdev_continue_rcu(net, d)		\
	list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
2749 2750
#define for_each_netdev_in_bond_rcu(bond, slave)	\
		for_each_netdev_rcu(&init_net, slave)	\
2751
			if (netdev_master_upper_dev_get_rcu(slave) == (bond))
2752
#define net_device_entry(lh)	list_entry(lh, struct net_device, dev_list)
2753

2754 2755 2756 2757 2758
static inline struct net_device *next_net_device(struct net_device *dev)
{
	struct list_head *lh;
	struct net *net;

2759
	net = dev_net(dev);
2760 2761 2762 2763
	lh = dev->dev_list.next;
	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
}

2764 2765 2766 2767 2768 2769
static inline struct net_device *next_net_device_rcu(struct net_device *dev)
{
	struct list_head *lh;
	struct net *net;

	net = dev_net(dev);
E
Eric Dumazet 已提交
2770
	lh = rcu_dereference(list_next_rcu(&dev->dev_list));
2771 2772 2773
	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
}

2774 2775 2776 2777 2778
static inline struct net_device *first_net_device(struct net *net)
{
	return list_empty(&net->dev_base_head) ? NULL :
		net_device_entry(net->dev_base_head.next);
}
2779

E
Eric Dumazet 已提交
2780 2781 2782 2783 2784 2785 2786
static inline struct net_device *first_net_device_rcu(struct net *net)
{
	struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));

	return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
}

2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
int netdev_boot_setup_check(struct net_device *dev);
unsigned long netdev_boot_base(const char *prefix, int unit);
struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
				       const char *hwaddr);
struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type);
void dev_add_pack(struct packet_type *pt);
void dev_remove_pack(struct packet_type *pt);
void __dev_remove_pack(struct packet_type *pt);
void dev_add_offload(struct packet_offload *po);
void dev_remove_offload(struct packet_offload *po);

2799
int dev_get_iflink(const struct net_device *dev);
2800
int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
2801 2802
struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
				      unsigned short mask);
2803 2804 2805 2806
struct net_device *dev_get_by_name(struct net *net, const char *name);
struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
struct net_device *__dev_get_by_name(struct net *net, const char *name);
int dev_alloc_name(struct net_device *dev, const char *name);
2807
int dev_open(struct net_device *dev, struct netlink_ext_ack *extack);
2808 2809
void dev_close(struct net_device *dev);
void dev_close_many(struct list_head *head, bool unlink);
2810
void dev_disable_lro(struct net_device *dev);
2811
int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
2812
u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
2813
		     struct net_device *sb_dev);
2814
u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
2815
		       struct net_device *sb_dev);
2816
int dev_queue_xmit(struct sk_buff *skb);
2817
int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev);
2818
int dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
2819 2820 2821
int register_netdevice(struct net_device *dev);
void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
void unregister_netdevice_many(struct list_head *head);
2822 2823 2824 2825 2826
static inline void unregister_netdevice(struct net_device *dev)
{
	unregister_netdevice_queue(dev, NULL);
}

2827 2828
int netdev_refcnt_read(const struct net_device *dev);
void free_netdev(struct net_device *dev);
2829
void netdev_freemem(struct net_device *dev);
2830
int init_dummy_netdev(struct net_device *dev);
2831

2832 2833 2834
struct net_device *netdev_get_xmit_slave(struct net_device *dev,
					 struct sk_buff *skb,
					 bool all_slaves);
2835 2836 2837
struct net_device *dev_get_by_index(struct net *net, int ifindex);
struct net_device *__dev_get_by_index(struct net *net, int ifindex);
struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
2838
struct net_device *dev_get_by_napi_id(unsigned int napi_id);
2839 2840
int netdev_get_name(struct net *net, char *name, int ifindex);
int dev_restart(struct net_device *dev);
2841
int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
2842
int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb);
2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858

static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
{
	return NAPI_GRO_CB(skb)->data_offset;
}

static inline unsigned int skb_gro_len(const struct sk_buff *skb)
{
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
}

static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
{
	NAPI_GRO_CB(skb)->data_offset += len;
}

2859 2860
static inline void *skb_gro_header_fast(struct sk_buff *skb,
					unsigned int offset)
2861
{
2862 2863
	return NAPI_GRO_CB(skb)->frag0 + offset;
}
2864

2865 2866 2867 2868
static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
{
	return NAPI_GRO_CB(skb)->frag0_len < hlen;
}
2869

2870 2871 2872 2873 2874 2875
static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
{
	NAPI_GRO_CB(skb)->frag0 = NULL;
	NAPI_GRO_CB(skb)->frag0_len = 0;
}

2876 2877 2878
static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
					unsigned int offset)
{
2879 2880 2881
	if (!pskb_may_pull(skb, hlen))
		return NULL;

2882
	skb_gro_frag0_invalidate(skb);
2883
	return skb->data + offset;
2884
}
L
Linus Torvalds 已提交
2885

H
Herbert Xu 已提交
2886 2887
static inline void *skb_gro_network_header(struct sk_buff *skb)
{
2888 2889
	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
	       skb_network_offset(skb);
H
Herbert Xu 已提交
2890 2891
}

2892 2893 2894
static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
					const void *start, unsigned int len)
{
T
Tom Herbert 已提交
2895
	if (NAPI_GRO_CB(skb)->csum_valid)
2896 2897 2898 2899
		NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum,
						  csum_partial(start, len, 0));
}

T
Tom Herbert 已提交
2900 2901 2902 2903 2904 2905 2906
/* GRO checksum functions. These are logical equivalents of the normal
 * checksum functions (in skbuff.h) except that they operate on the GRO
 * offsets and fields in sk_buff.
 */

__sum16 __skb_gro_checksum_complete(struct sk_buff *skb);

2907 2908
static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
{
2909
	return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
2910 2911
}

T
Tom Herbert 已提交
2912 2913 2914 2915
static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
						      bool zero_okay,
						      __sum16 check)
{
2916 2917 2918
	return ((skb->ip_summed != CHECKSUM_PARTIAL ||
		skb_checksum_start_offset(skb) <
		 skb_gro_offset(skb)) &&
2919
		!skb_at_gro_remcsum_start(skb) &&
2920
		NAPI_GRO_CB(skb)->csum_cnt == 0 &&
T
Tom Herbert 已提交
2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937
		(!zero_okay || check));
}

static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
							   __wsum psum)
{
	if (NAPI_GRO_CB(skb)->csum_valid &&
	    !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
		return 0;

	NAPI_GRO_CB(skb)->csum = psum;

	return __skb_gro_checksum_complete(skb);
}

static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
{
2938 2939 2940 2941 2942 2943 2944 2945 2946
	if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
		/* Consume a checksum from CHECKSUM_UNNECESSARY */
		NAPI_GRO_CB(skb)->csum_cnt--;
	} else {
		/* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
		 * verified a new top level checksum or an encapsulated one
		 * during GRO. This saves work if we fallback to normal path.
		 */
		__skb_incr_checksum_unnecessary(skb);
T
Tom Herbert 已提交
2947 2948 2949 2950 2951 2952 2953 2954 2955 2956
	}
}

#define __skb_gro_checksum_validate(skb, proto, zero_okay, check,	\
				    compute_pseudo)			\
({									\
	__sum16 __ret = 0;						\
	if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))	\
		__ret = __skb_gro_checksum_validate_complete(skb,	\
				compute_pseudo(skb, proto));		\
2957
	if (!__ret)							\
T
Tom Herbert 已提交
2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971
		skb_gro_incr_csum_unnecessary(skb);			\
	__ret;								\
})

#define skb_gro_checksum_validate(skb, proto, compute_pseudo)		\
	__skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)

#define skb_gro_checksum_validate_zero_check(skb, proto, check,		\
					     compute_pseudo)		\
	__skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)

#define skb_gro_checksum_simple_validate(skb)				\
	__skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)

2972 2973 2974 2975 2976 2977 2978
static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
{
	return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
		!NAPI_GRO_CB(skb)->csum_valid);
}

static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
2979
					      __wsum pseudo)
2980 2981 2982 2983 2984
{
	NAPI_GRO_CB(skb)->csum = ~pseudo;
	NAPI_GRO_CB(skb)->csum_valid = 1;
}

2985
#define skb_gro_checksum_try_convert(skb, proto, compute_pseudo)	\
2986 2987
do {									\
	if (__skb_gro_checksum_convert_check(skb))			\
2988
		__skb_gro_checksum_convert(skb, 			\
2989 2990 2991
					   compute_pseudo(skb, proto));	\
} while (0)

2992 2993 2994 2995 2996 2997 2998
struct gro_remcsum {
	int offset;
	__wsum delta;
};

static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
{
2999
	grc->offset = 0;
3000 3001 3002
	grc->delta = 0;
}

3003 3004 3005 3006 3007
static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
					    unsigned int off, size_t hdrlen,
					    int start, int offset,
					    struct gro_remcsum *grc,
					    bool nopartial)
3008 3009
{
	__wsum delta;
3010
	size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
3011 3012 3013

	BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);

3014
	if (!nopartial) {
3015 3016 3017 3018 3019 3020 3021 3022 3023
		NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
		return ptr;
	}

	ptr = skb_gro_header_fast(skb, off);
	if (skb_gro_header_hard(skb, off + plen)) {
		ptr = skb_gro_header_slow(skb, off + plen, off);
		if (!ptr)
			return NULL;
3024 3025
	}

3026 3027
	delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
			       start, offset);
3028 3029 3030

	/* Adjust skb->csum since we changed the packet */
	NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
3031

3032
	grc->offset = off + hdrlen + offset;
3033
	grc->delta = delta;
3034 3035

	return ptr;
3036 3037
}

3038 3039 3040
static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
					   struct gro_remcsum *grc)
{
3041 3042 3043
	void *ptr;
	size_t plen = grc->offset + sizeof(u16);

3044 3045 3046
	if (!grc->delta)
		return;

3047 3048 3049 3050 3051 3052 3053 3054
	ptr = skb_gro_header_fast(skb, grc->offset);
	if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
		ptr = skb_gro_header_slow(skb, plen, grc->offset);
		if (!ptr)
			return;
	}

	remcsum_unadjust((__sum16 *)ptr, grc->delta);
3055
}
3056

3057
#ifdef CONFIG_XFRM_OFFLOAD
3058
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
3059 3060 3061 3062
{
	if (PTR_ERR(pp) != -EINPROGRESS)
		NAPI_GRO_CB(skb)->flush |= flush;
}
3063
static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
3064
					       struct sk_buff *pp,
3065 3066 3067 3068 3069 3070 3071 3072 3073
					       int flush,
					       struct gro_remcsum *grc)
{
	if (PTR_ERR(pp) != -EINPROGRESS) {
		NAPI_GRO_CB(skb)->flush |= flush;
		skb_gro_remcsum_cleanup(skb, grc);
		skb->remcsum_offload = 0;
	}
}
3074
#else
3075
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
3076 3077 3078
{
	NAPI_GRO_CB(skb)->flush |= flush;
}
3079
static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
3080
					       struct sk_buff *pp,
3081 3082 3083 3084 3085 3086 3087
					       int flush,
					       struct gro_remcsum *grc)
{
	NAPI_GRO_CB(skb)->flush |= flush;
	skb_gro_remcsum_cleanup(skb, grc);
	skb->remcsum_offload = 0;
}
3088
#endif
3089

3090 3091
static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
				  unsigned short type,
3092
				  const void *daddr, const void *saddr,
3093
				  unsigned int len)
3094
{
3095
	if (!dev->header_ops || !dev->header_ops->create)
3096
		return 0;
3097 3098

	return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
3099 3100
}

S
Stephen Hemminger 已提交
3101 3102 3103 3104 3105
static inline int dev_parse_header(const struct sk_buff *skb,
				   unsigned char *haddr)
{
	const struct net_device *dev = skb->dev;

3106
	if (!dev->header_ops || !dev->header_ops->parse)
S
Stephen Hemminger 已提交
3107
		return 0;
3108
	return dev->header_ops->parse(skb, haddr);
S
Stephen Hemminger 已提交
3109 3110
}

3111 3112 3113 3114 3115 3116 3117 3118 3119
static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
{
	const struct net_device *dev = skb->dev;

	if (!dev->header_ops || !dev->header_ops->parse_protocol)
		return 0;
	return dev->header_ops->parse_protocol(skb);
}

3120 3121 3122 3123 3124 3125
/* ll_header must have at least hard_header_len allocated */
static inline bool dev_validate_header(const struct net_device *dev,
				       char *ll_header, int len)
{
	if (likely(len >= dev->hard_header_len))
		return true;
3126 3127
	if (len < dev->min_header_len)
		return false;
3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139

	if (capable(CAP_SYS_RAWIO)) {
		memset(ll_header + len, 0, dev->hard_header_len - len);
		return true;
	}

	if (dev->header_ops && dev->header_ops->validate)
		return dev->header_ops->validate(ll_header, len);

	return false;
}

3140 3141
typedef int gifconf_func_t(struct net_device * dev, char __user * bufptr,
			   int len, int size);
3142
int register_gifconf(unsigned int family, gifconf_func_t *gifconf);
L
Linus Torvalds 已提交
3143 3144 3145 3146 3147
static inline int unregister_gifconf(unsigned int family)
{
	return register_gifconf(family, NULL);
}

3148
#ifdef CONFIG_NET_FLOW_LIMIT
3149
#define FLOW_LIMIT_HISTORY	(1 << 7)  /* must be ^2 and !overflow buckets */
3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160
struct sd_flow_limit {
	u64			count;
	unsigned int		num_buckets;
	unsigned int		history_head;
	u16			history[FLOW_LIMIT_HISTORY];
	u8			buckets[];
};

extern int netdev_flow_limit_table_len;
#endif /* CONFIG_NET_FLOW_LIMIT */

L
Linus Torvalds 已提交
3161
/*
B
Bjorn Helgaas 已提交
3162
 * Incoming packets are placed on per-CPU queues
L
Linus Torvalds 已提交
3163
 */
E
Eric Dumazet 已提交
3164
struct softnet_data {
L
Linus Torvalds 已提交
3165
	struct list_head	poll_list;
3166
	struct sk_buff_head	process_queue;
L
Linus Torvalds 已提交
3167

C
Changli Gao 已提交
3168
	/* stats */
3169 3170 3171
	unsigned int		processed;
	unsigned int		time_squeeze;
	unsigned int		received_rps;
3172
#ifdef CONFIG_RPS
E
Eric Dumazet 已提交
3173
	struct softnet_data	*rps_ipi_list;
E
Eric Dumazet 已提交
3174 3175 3176 3177 3178 3179 3180
#endif
#ifdef CONFIG_NET_FLOW_LIMIT
	struct sd_flow_limit __rcu *flow_limit;
#endif
	struct Qdisc		*output_queue;
	struct Qdisc		**output_queue_tailp;
	struct sk_buff		*completion_queue;
3181 3182 3183
#ifdef CONFIG_XFRM_OFFLOAD
	struct sk_buff_head	xfrm_backlog;
#endif
3184 3185 3186 3187 3188
	/* written and read only by owning cpu: */
	struct {
		u16 recursion;
		u8  more;
	} xmit;
E
Eric Dumazet 已提交
3189
#ifdef CONFIG_RPS
3190 3191 3192 3193 3194 3195
	/* input_queue_head should be written by cpu owning this struct,
	 * and only read by other cpus. Worth using a cache line.
	 */
	unsigned int		input_queue_head ____cacheline_aligned_in_smp;

	/* Elements below can be accessed between CPUs for RPS/RFS */
3196
	call_single_data_t	csd ____cacheline_aligned_in_smp;
E
Eric Dumazet 已提交
3197 3198
	struct softnet_data	*rps_ipi_next;
	unsigned int		cpu;
3199
	unsigned int		input_queue_tail;
3200
#endif
3201
	unsigned int		dropped;
T
Tom Herbert 已提交
3202
	struct sk_buff_head	input_pkt_queue;
3203
	struct napi_struct	backlog;
3204

L
Linus Torvalds 已提交
3205 3206
};

3207
static inline void input_queue_head_incr(struct softnet_data *sd)
T
Tom Herbert 已提交
3208 3209
{
#ifdef CONFIG_RPS
3210 3211 3212 3213 3214 3215 3216 3217 3218
	sd->input_queue_head++;
#endif
}

static inline void input_queue_tail_incr_save(struct softnet_data *sd,
					      unsigned int *qtail)
{
#ifdef CONFIG_RPS
	*qtail = ++sd->input_queue_tail;
T
Tom Herbert 已提交
3219 3220 3221
#endif
}

T
Tom Herbert 已提交
3222
DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
L
Linus Torvalds 已提交
3223

3224 3225
static inline int dev_recursion_level(void)
{
3226
	return this_cpu_read(softnet_data.xmit.recursion);
3227 3228
}

3229
#define XMIT_RECURSION_LIMIT	8
3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245
static inline bool dev_xmit_recursion(void)
{
	return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
			XMIT_RECURSION_LIMIT);
}

static inline void dev_xmit_recursion_inc(void)
{
	__this_cpu_inc(softnet_data.xmit.recursion);
}

static inline void dev_xmit_recursion_dec(void)
{
	__this_cpu_dec(softnet_data.xmit.recursion);
}

3246
void __netif_schedule(struct Qdisc *q);
3247
void netif_schedule_queue(struct netdev_queue *txq);
3248

3249 3250 3251 3252 3253 3254 3255 3256
static inline void netif_tx_schedule_all(struct net_device *dev)
{
	unsigned int i;

	for (i = 0; i < dev->num_tx_queues; i++)
		netif_schedule_queue(netdev_get_tx_queue(dev, i));
}

3257
static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
3258
{
3259
	clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3260 3261
}

3262 3263 3264 3265 3266 3267
/**
 *	netif_start_queue - allow transmit
 *	@dev: network device
 *
 *	Allow upper layers to call the device hard_start_xmit routine.
 */
L
Linus Torvalds 已提交
3268 3269
static inline void netif_start_queue(struct net_device *dev)
{
3270
	netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
L
Linus Torvalds 已提交
3271 3272
}

3273 3274 3275 3276 3277 3278 3279 3280 3281 3282
static inline void netif_tx_start_all_queues(struct net_device *dev)
{
	unsigned int i;

	for (i = 0; i < dev->num_tx_queues; i++) {
		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
		netif_tx_start_queue(txq);
	}
}

3283
void netif_tx_wake_queue(struct netdev_queue *dev_queue);
3284

3285 3286 3287 3288 3289 3290 3291
/**
 *	netif_wake_queue - restart transmit
 *	@dev: network device
 *
 *	Allow upper layers to call the device hard_start_xmit routine.
 *	Used for flow control when transmit resources are available.
 */
3292 3293
static inline void netif_wake_queue(struct net_device *dev)
{
3294
	netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
L
Linus Torvalds 已提交
3295 3296
}

3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
static inline void netif_tx_wake_all_queues(struct net_device *dev)
{
	unsigned int i;

	for (i = 0; i < dev->num_tx_queues; i++) {
		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
		netif_tx_wake_queue(txq);
	}
}

3307
static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
3308
{
3309
	set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3310 3311
}

3312 3313 3314 3315 3316 3317 3318
/**
 *	netif_stop_queue - stop transmitted packets
 *	@dev: network device
 *
 *	Stop upper layers calling the device hard_start_xmit routine.
 *	Used for flow control when transmit resources are unavailable.
 */
L
Linus Torvalds 已提交
3319 3320
static inline void netif_stop_queue(struct net_device *dev)
{
3321
	netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
L
Linus Torvalds 已提交
3322 3323
}

3324
void netif_tx_stop_all_queues(struct net_device *dev);
3325

3326
static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
3327
{
3328
	return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3329 3330
}

3331 3332 3333 3334 3335 3336
/**
 *	netif_queue_stopped - test if transmit queue is flowblocked
 *	@dev: network device
 *
 *	Test if transmit queue on device is currently unable to send.
 */
3337
static inline bool netif_queue_stopped(const struct net_device *dev)
L
Linus Torvalds 已提交
3338
{
3339
	return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
L
Linus Torvalds 已提交
3340 3341
}

3342
static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
3343
{
3344 3345 3346
	return dev_queue->state & QUEUE_STATE_ANY_XOFF;
}

3347 3348
static inline bool
netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
3349 3350 3351 3352
{
	return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
}

3353 3354 3355 3356 3357 3358
static inline bool
netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
{
	return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
}

3359 3360 3361 3362 3363
/**
 *	netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
 *	@dev_queue: pointer to transmit queue
 *
 * BQL enabled drivers might use this helper in their ndo_start_xmit(),
B
Bjorn Helgaas 已提交
3364
 * to give appropriate hint to the CPU.
3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377
 */
static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
{
#ifdef CONFIG_BQL
	prefetchw(&dev_queue->dql.num_queued);
#endif
}

/**
 *	netdev_txq_bql_complete_prefetchw - prefetch bql data for write
 *	@dev_queue: pointer to transmit queue
 *
 * BQL enabled drivers might use this helper in their TX completion path,
B
Bjorn Helgaas 已提交
3378
 * to give appropriate hint to the CPU.
3379 3380 3381 3382 3383 3384 3385 3386
 */
static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
{
#ifdef CONFIG_BQL
	prefetchw(&dev_queue->dql.limit);
#endif
}

3387 3388 3389
static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
					unsigned int bytes)
{
T
Tom Herbert 已提交
3390 3391
#ifdef CONFIG_BQL
	dql_queued(&dev_queue->dql, bytes);
3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407

	if (likely(dql_avail(&dev_queue->dql) >= 0))
		return;

	set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);

	/*
	 * The XOFF flag must be set before checking the dql_avail below,
	 * because in netdev_tx_completed_queue we update the dql_completed
	 * before checking the XOFF flag.
	 */
	smp_mb();

	/* check again in case another CPU has just made room avail */
	if (unlikely(dql_avail(&dev_queue->dql) >= 0))
		clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
T
Tom Herbert 已提交
3408
#endif
3409 3410
}

3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430
/* Variant of netdev_tx_sent_queue() for drivers that are aware
 * that they should not test BQL status themselves.
 * We do want to change __QUEUE_STATE_STACK_XOFF only for the last
 * skb of a batch.
 * Returns true if the doorbell must be used to kick the NIC.
 */
static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
					  unsigned int bytes,
					  bool xmit_more)
{
	if (xmit_more) {
#ifdef CONFIG_BQL
		dql_queued(&dev_queue->dql, bytes);
#endif
		return netif_tx_queue_stopped(dev_queue);
	}
	netdev_tx_sent_queue(dev_queue, bytes);
	return true;
}

3431 3432 3433 3434 3435 3436 3437 3438 3439
/**
 * 	netdev_sent_queue - report the number of bytes queued to hardware
 * 	@dev: network device
 * 	@bytes: number of bytes queued to the hardware device queue
 *
 * 	Report the number of bytes queued for sending/completion to the network
 * 	device hardware queue. @bytes should be a good approximation and should
 * 	exactly match netdev_completed_queue() @bytes
 */
3440 3441 3442 3443 3444
static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
{
	netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
}

3445 3446 3447 3448 3449 3450 3451 3452
static inline bool __netdev_sent_queue(struct net_device *dev,
				       unsigned int bytes,
				       bool xmit_more)
{
	return __netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes,
				      xmit_more);
}

3453
static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3454
					     unsigned int pkts, unsigned int bytes)
3455
{
T
Tom Herbert 已提交
3456
#ifdef CONFIG_BQL
3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468
	if (unlikely(!bytes))
		return;

	dql_completed(&dev_queue->dql, bytes);

	/*
	 * Without the memory barrier there is a small possiblity that
	 * netdev_tx_sent_queue will miss the update and cause the queue to
	 * be stopped forever
	 */
	smp_mb();

3469
	if (unlikely(dql_avail(&dev_queue->dql) < 0))
3470 3471 3472 3473
		return;

	if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
		netif_schedule_queue(dev_queue);
T
Tom Herbert 已提交
3474
#endif
3475 3476
}

3477 3478 3479 3480 3481 3482 3483 3484 3485 3486
/**
 * 	netdev_completed_queue - report bytes and packets completed by device
 * 	@dev: network device
 * 	@pkts: actual number of packets sent over the medium
 * 	@bytes: actual number of bytes sent over the medium
 *
 * 	Report the number of bytes and packets transmitted by the network device
 * 	hardware queue over the physical medium, @bytes must exactly match the
 * 	@bytes amount passed to netdev_sent_queue()
 */
3487
static inline void netdev_completed_queue(struct net_device *dev,
3488
					  unsigned int pkts, unsigned int bytes)
3489 3490 3491 3492 3493 3494
{
	netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
}

static inline void netdev_tx_reset_queue(struct netdev_queue *q)
{
T
Tom Herbert 已提交
3495
#ifdef CONFIG_BQL
3496
	clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
T
Tom Herbert 已提交
3497 3498
	dql_reset(&q->dql);
#endif
3499 3500
}

3501 3502 3503 3504 3505 3506 3507
/**
 * 	netdev_reset_queue - reset the packets and bytes count of a network device
 * 	@dev_queue: network device
 *
 * 	Reset the bytes and packet count of a network device and clear the
 * 	software flow control OFF bit for this network device
 */
3508 3509 3510
static inline void netdev_reset_queue(struct net_device *dev_queue)
{
	netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
3511 3512
}

3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532
/**
 * 	netdev_cap_txqueue - check if selected tx queue exceeds device queues
 * 	@dev: network device
 * 	@queue_index: given tx queue index
 *
 * 	Returns 0 if given tx queue index >= number of device tx queues,
 * 	otherwise returns the originally passed tx queue index.
 */
static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
{
	if (unlikely(queue_index >= dev->real_num_tx_queues)) {
		net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
				     dev->name, queue_index,
				     dev->real_num_tx_queues);
		return 0;
	}

	return queue_index;
}

3533 3534 3535 3536 3537 3538
/**
 *	netif_running - test if up
 *	@dev: network device
 *
 *	Test if the device has been brought up.
 */
3539
static inline bool netif_running(const struct net_device *dev)
L
Linus Torvalds 已提交
3540 3541 3542 3543
{
	return test_bit(__LINK_STATE_START, &dev->state);
}

3544
/*
B
Bjorn Helgaas 已提交
3545
 * Routines to manage the subqueues on a device.  We only need start,
3546 3547 3548 3549
 * stop, and a check if it's stopped.  All other device management is
 * done at the overall netdevice level.
 * Also test the device if we're multiqueue.
 */
3550 3551 3552 3553 3554 3555 3556 3557

/**
 *	netif_start_subqueue - allow sending packets on subqueue
 *	@dev: network device
 *	@queue_index: sub queue index
 *
 * Start individual transmit queue of a device with multiple transmit queues.
 */
3558 3559
static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
{
3560
	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3561 3562

	netif_tx_start_queue(txq);
3563 3564
}

3565 3566 3567 3568 3569 3570 3571
/**
 *	netif_stop_subqueue - stop sending packets on subqueue
 *	@dev: network device
 *	@queue_index: sub queue index
 *
 * Stop individual transmit queue of a device with multiple transmit queues.
 */
3572 3573
static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
{
3574
	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3575
	netif_tx_stop_queue(txq);
3576 3577
}

3578 3579 3580 3581 3582 3583 3584
/**
 *	netif_subqueue_stopped - test status of subqueue
 *	@dev: network device
 *	@queue_index: sub queue index
 *
 * Check individual transmit queue of a device with multiple transmit queues.
 */
3585 3586
static inline bool __netif_subqueue_stopped(const struct net_device *dev,
					    u16 queue_index)
3587
{
3588
	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3589 3590

	return netif_tx_queue_stopped(txq);
3591 3592
}

3593 3594
static inline bool netif_subqueue_stopped(const struct net_device *dev,
					  struct sk_buff *skb)
3595 3596 3597
{
	return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
}
3598

3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611
/**
 *	netif_wake_subqueue - allow sending packets on subqueue
 *	@dev: network device
 *	@queue_index: sub queue index
 *
 * Resume individual transmit queue of a device with multiple transmit queues.
 */
static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
{
	struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);

	netif_tx_wake_queue(txq);
}
3612

3613
#ifdef CONFIG_XPS
3614
int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
3615
			u16 index);
3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676
int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
			  u16 index, bool is_rxqs_map);

/**
 *	netif_attr_test_mask - Test a CPU or Rx queue set in a mask
 *	@j: CPU/Rx queue index
 *	@mask: bitmask of all cpus/rx queues
 *	@nr_bits: number of bits in the bitmask
 *
 * Test if a CPU or Rx queue index is set in a mask of all CPU/Rx queues.
 */
static inline bool netif_attr_test_mask(unsigned long j,
					const unsigned long *mask,
					unsigned int nr_bits)
{
	cpu_max_bits_warn(j, nr_bits);
	return test_bit(j, mask);
}

/**
 *	netif_attr_test_online - Test for online CPU/Rx queue
 *	@j: CPU/Rx queue index
 *	@online_mask: bitmask for CPUs/Rx queues that are online
 *	@nr_bits: number of bits in the bitmask
 *
 * Returns true if a CPU/Rx queue is online.
 */
static inline bool netif_attr_test_online(unsigned long j,
					  const unsigned long *online_mask,
					  unsigned int nr_bits)
{
	cpu_max_bits_warn(j, nr_bits);

	if (online_mask)
		return test_bit(j, online_mask);

	return (j < nr_bits);
}

/**
 *	netif_attrmask_next - get the next CPU/Rx queue in a cpu/Rx queues mask
 *	@n: CPU/Rx queue index
 *	@srcp: the cpumask/Rx queue mask pointer
 *	@nr_bits: number of bits in the bitmask
 *
 * Returns >= nr_bits if no further CPUs/Rx queues set.
 */
static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
					       unsigned int nr_bits)
{
	/* -1 is a legal arg here. */
	if (n != -1)
		cpu_max_bits_warn(n, nr_bits);

	if (srcp)
		return find_next_bit(srcp, nr_bits, n + 1);

	return n + 1;
}

/**
3677
 *	netif_attrmask_next_and - get the next CPU/Rx queue in \*src1p & \*src2p
3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701
 *	@n: CPU/Rx queue index
 *	@src1p: the first CPUs/Rx queues mask pointer
 *	@src2p: the second CPUs/Rx queues mask pointer
 *	@nr_bits: number of bits in the bitmask
 *
 * Returns >= nr_bits if no further CPUs/Rx queues set in both.
 */
static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
					  const unsigned long *src2p,
					  unsigned int nr_bits)
{
	/* -1 is a legal arg here. */
	if (n != -1)
		cpu_max_bits_warn(n, nr_bits);

	if (src1p && src2p)
		return find_next_and_bit(src1p, src2p, nr_bits, n + 1);
	else if (src1p)
		return find_next_bit(src1p, nr_bits, n + 1);
	else if (src2p)
		return find_next_bit(src2p, nr_bits, n + 1);

	return n + 1;
}
3702 3703
#else
static inline int netif_set_xps_queue(struct net_device *dev,
3704
				      const struct cpumask *mask,
3705 3706 3707 3708
				      u16 index)
{
	return 0;
}
3709 3710 3711 3712 3713 3714 3715

static inline int __netif_set_xps_queue(struct net_device *dev,
					const unsigned long *mask,
					u16 index, bool is_rxqs_map)
{
	return 0;
}
3716 3717
#endif

3718 3719 3720 3721 3722 3723
/**
 *	netif_is_multiqueue - test if device has multiple transmit queues
 *	@dev: network device
 *
 * Check if device has multiple transmit queues
 */
3724
static inline bool netif_is_multiqueue(const struct net_device *dev)
3725
{
E
Eric Dumazet 已提交
3726
	return dev->num_tx_queues > 1;
3727
}
L
Linus Torvalds 已提交
3728

3729
int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
3730

3731
#ifdef CONFIG_SYSFS
3732
int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
3733 3734
#else
static inline int netif_set_real_num_rx_queues(struct net_device *dev,
3735
						unsigned int rxqs)
3736
{
3737
	dev->real_num_rx_queues = rxqs;
3738 3739 3740 3741
	return 0;
}
#endif

3742 3743 3744 3745 3746 3747
static inline struct netdev_rx_queue *
__netif_get_rx_queue(struct net_device *dev, unsigned int rxq)
{
	return dev->_rx + rxq;
}

3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759
#ifdef CONFIG_SYSFS
static inline unsigned int get_netdev_rx_queue_index(
		struct netdev_rx_queue *queue)
{
	struct net_device *dev = queue->dev;
	int index = queue - dev->_rx;

	BUG_ON(index >= dev->num_rx_queues);
	return index;
}
#endif

3760
#define DEFAULT_MAX_NUM_RSS_QUEUES	(8)
3761
int netif_get_num_default_rss_queues(void);
3762

3763 3764 3765 3766 3767 3768 3769
enum skb_free_reason {
	SKB_REASON_CONSUMED,
	SKB_REASON_DROPPED,
};

void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
L
Linus Torvalds 已提交
3770

3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788
/*
 * It is not allowed to call kfree_skb() or consume_skb() from hardware
 * interrupt context or with hardware interrupts being disabled.
 * (in_irq() || irqs_disabled())
 *
 * We provide four helpers that can be used in following contexts :
 *
 * dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
 *  replacing kfree_skb(skb)
 *
 * dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
 *  Typically used in place of consume_skb(skb) in TX completion path
 *
 * dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
 *  replacing kfree_skb(skb)
 *
 * dev_consume_skb_any(skb) when caller doesn't know its current irq context,
 *  and consumed a packet. Used in place of consume_skb(skb)
L
Linus Torvalds 已提交
3789
 */
3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
static inline void dev_kfree_skb_irq(struct sk_buff *skb)
{
	__dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
}

static inline void dev_consume_skb_irq(struct sk_buff *skb)
{
	__dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
}

static inline void dev_kfree_skb_any(struct sk_buff *skb)
{
	__dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
}

static inline void dev_consume_skb_any(struct sk_buff *skb)
{
	__dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
}
L
Linus Torvalds 已提交
3809

3810 3811
void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
3812 3813
int netif_rx(struct sk_buff *skb);
int netif_rx_ni(struct sk_buff *skb);
3814
int netif_rx_any_context(struct sk_buff *skb);
3815
int netif_receive_skb(struct sk_buff *skb);
3816
int netif_receive_skb_core(struct sk_buff *skb);
3817
void netif_receive_skb_list(struct list_head *head);
3818 3819 3820 3821
gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
void napi_gro_flush(struct napi_struct *napi, bool flush_old);
struct sk_buff *napi_get_frags(struct napi_struct *napi);
gro_result_t napi_gro_frags(struct napi_struct *napi);
3822 3823
struct packet_offload *gro_find_receive_by_type(__be16 type);
struct packet_offload *gro_find_complete_by_type(__be16 type);
3824 3825 3826 3827 3828 3829 3830

static inline void napi_free_frags(struct napi_struct *napi)
{
	kfree_skb(napi->skb);
	napi->skb = NULL;
}

M
Mahesh Bandewar 已提交
3831
bool netdev_is_rx_handler_busy(struct net_device *dev);
3832 3833 3834 3835 3836 3837
int netdev_rx_handler_register(struct net_device *dev,
			       rx_handler_func_t *rx_handler,
			       void *rx_handler_data);
void netdev_rx_handler_unregister(struct net_device *dev);

bool dev_valid_name(const char *name);
3838 3839
int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
		bool *need_copyout);
3840
int dev_ifconf(struct net *net, struct ifconf *, int);
3841 3842
int dev_ethtool(struct net *net, struct ifreq *);
unsigned int dev_get_flags(const struct net_device *);
3843 3844
int __dev_change_flags(struct net_device *dev, unsigned int flags,
		       struct netlink_ext_ack *extack);
3845 3846
int dev_change_flags(struct net_device *dev, unsigned int flags,
		     struct netlink_ext_ack *extack);
3847 3848
void __dev_notify_flags(struct net_device *, unsigned int old_flags,
			unsigned int gchanges);
3849 3850
int dev_change_name(struct net_device *, const char *);
int dev_set_alias(struct net_device *, const char *, size_t);
3851
int dev_get_alias(const struct net_device *, char *, size_t);
3852
int dev_change_net_namespace(struct net_device *, struct net *, const char *);
3853
int __dev_set_mtu(struct net_device *, int);
3854 3855
int dev_validate_mtu(struct net_device *dev, int mtu,
		     struct netlink_ext_ack *extack);
3856 3857
int dev_set_mtu_ext(struct net_device *dev, int mtu,
		    struct netlink_ext_ack *extack);
3858
int dev_set_mtu(struct net_device *, int);
3859
int dev_change_tx_queue_len(struct net_device *, unsigned long);
3860
void dev_set_group(struct net_device *, int);
3861 3862
int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
			      struct netlink_ext_ack *extack);
3863 3864
int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
			struct netlink_ext_ack *extack);
3865 3866
int dev_change_carrier(struct net_device *, bool new_carrier);
int dev_get_phys_port_id(struct net_device *dev,
3867
			 struct netdev_phys_item_id *ppid);
3868 3869
int dev_get_phys_port_name(struct net_device *dev,
			   char *name, size_t len);
3870 3871 3872
int dev_get_port_parent_id(struct net_device *dev,
			   struct netdev_phys_item_id *ppid, bool recurse);
bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b);
3873
int dev_change_proto_down(struct net_device *dev, bool proto_down);
3874
int dev_change_proto_down_generic(struct net_device *dev, bool proto_down);
3875 3876
void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
				  u32 value);
3877
struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
3878 3879
struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
				    struct netdev_queue *txq, int *ret);
3880

3881
typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
3882
int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
3883
		      int fd, int expected_fd, u32 flags);
3884
int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
3885
u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
3886

3887
int xdp_umem_query(struct net_device *dev, u16 queue_id);
3888

H
Herbert Xu 已提交
3889
int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3890
int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3891 3892
bool is_skb_forwardable(const struct net_device *dev,
			const struct sk_buff *skb);
L
Linus Torvalds 已提交
3893

3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908
static __always_inline int ____dev_forward_skb(struct net_device *dev,
					       struct sk_buff *skb)
{
	if (skb_orphan_frags(skb, GFP_ATOMIC) ||
	    unlikely(!is_skb_forwardable(dev, skb))) {
		atomic_long_inc(&dev->rx_dropped);
		kfree_skb(skb);
		return NET_RX_DROP;
	}

	skb_scrub_packet(skb, true);
	skb->priority = 0;
	return 0;
}

3909
bool dev_nit_active(struct net_device *dev);
3910 3911
void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);

3912
extern int		netdev_budget;
3913
extern unsigned int	netdev_budget_usecs;
L
Linus Torvalds 已提交
3914 3915

/* Called by rtnetlink.c:rtnl_unlock() */
3916
void netdev_run_todo(void);
L
Linus Torvalds 已提交
3917

3918 3919 3920 3921
/**
 *	dev_put - release reference to device
 *	@dev: network device
 *
3922
 * Release reference to device to allow it to be freed.
3923
 */
L
Linus Torvalds 已提交
3924 3925
static inline void dev_put(struct net_device *dev)
{
3926
	this_cpu_dec(*dev->pcpu_refcnt);
L
Linus Torvalds 已提交
3927 3928
}

3929 3930 3931 3932
/**
 *	dev_hold - get reference to device
 *	@dev: network device
 *
3933
 * Hold reference to device to keep it from being freed.
3934
 */
3935 3936
static inline void dev_hold(struct net_device *dev)
{
3937
	this_cpu_inc(*dev->pcpu_refcnt);
3938
}
L
Linus Torvalds 已提交
3939 3940 3941 3942

/* Carrier loss detection, dial on demand. The functions netif_carrier_on
 * and _off may be called from IRQ context, but it is caller
 * who is responsible for serialization of these calls.
S
Stefan Rompf 已提交
3943 3944 3945 3946
 *
 * The name carrier is inappropriate, these functions should really be
 * called netif_lowerlayer_*() because they represent the state of any
 * kind of lower layer not just hardware media.
L
Linus Torvalds 已提交
3947 3948
 */

3949 3950 3951
void linkwatch_init_dev(struct net_device *dev);
void linkwatch_fire_event(struct net_device *dev);
void linkwatch_forget_dev(struct net_device *dev);
L
Linus Torvalds 已提交
3952

3953 3954 3955 3956 3957 3958
/**
 *	netif_carrier_ok - test if carrier present
 *	@dev: network device
 *
 * Check if carrier is present on device
 */
3959
static inline bool netif_carrier_ok(const struct net_device *dev)
L
Linus Torvalds 已提交
3960 3961 3962 3963
{
	return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
}

3964
unsigned long dev_trans_start(struct net_device *dev);
3965

3966
void __netdev_watchdog_up(struct net_device *dev);
L
Linus Torvalds 已提交
3967

3968
void netif_carrier_on(struct net_device *dev);
L
Linus Torvalds 已提交
3969

3970
void netif_carrier_off(struct net_device *dev);
L
Linus Torvalds 已提交
3971

3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983
/**
 *	netif_dormant_on - mark device as dormant.
 *	@dev: network device
 *
 * Mark device as dormant (as per RFC2863).
 *
 * The dormant state indicates that the relevant interface is not
 * actually in a condition to pass packets (i.e., it is not 'up') but is
 * in a "pending" state, waiting for some external event.  For "on-
 * demand" interfaces, this new state identifies the situation where the
 * interface is waiting for events to place it in the up state.
 */
S
Stefan Rompf 已提交
3984 3985 3986 3987 3988 3989
static inline void netif_dormant_on(struct net_device *dev)
{
	if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
		linkwatch_fire_event(dev);
}

3990 3991 3992 3993 3994 3995
/**
 *	netif_dormant_off - set device as not dormant.
 *	@dev: network device
 *
 * Device is not in dormant state.
 */
S
Stefan Rompf 已提交
3996 3997 3998 3999 4000 4001
static inline void netif_dormant_off(struct net_device *dev)
{
	if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
		linkwatch_fire_event(dev);
}

4002
/**
4003
 *	netif_dormant - test if device is dormant
4004 4005
 *	@dev: network device
 *
4006
 * Check if device is dormant.
4007
 */
4008
static inline bool netif_dormant(const struct net_device *dev)
S
Stefan Rompf 已提交
4009 4010 4011 4012 4013
{
	return test_bit(__LINK_STATE_DORMANT, &dev->state);
}


A
Andrew Lunn 已提交
4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053
/**
 *	netif_testing_on - mark device as under test.
 *	@dev: network device
 *
 * Mark device as under test (as per RFC2863).
 *
 * The testing state indicates that some test(s) must be performed on
 * the interface. After completion, of the test, the interface state
 * will change to up, dormant, or down, as appropriate.
 */
static inline void netif_testing_on(struct net_device *dev)
{
	if (!test_and_set_bit(__LINK_STATE_TESTING, &dev->state))
		linkwatch_fire_event(dev);
}

/**
 *	netif_testing_off - set device as not under test.
 *	@dev: network device
 *
 * Device is not in testing state.
 */
static inline void netif_testing_off(struct net_device *dev)
{
	if (test_and_clear_bit(__LINK_STATE_TESTING, &dev->state))
		linkwatch_fire_event(dev);
}

/**
 *	netif_testing - test if device is under test
 *	@dev: network device
 *
 * Check if device is under test
 */
static inline bool netif_testing(const struct net_device *dev)
{
	return test_bit(__LINK_STATE_TESTING, &dev->state);
}


4054 4055 4056 4057 4058 4059
/**
 *	netif_oper_up - test if device is operational
 *	@dev: network device
 *
 * Check if carrier is operational
 */
4060
static inline bool netif_oper_up(const struct net_device *dev)
E
Eric Dumazet 已提交
4061
{
S
Stefan Rompf 已提交
4062 4063 4064 4065
	return (dev->operstate == IF_OPER_UP ||
		dev->operstate == IF_OPER_UNKNOWN /* backward compat */);
}

4066 4067 4068 4069 4070 4071
/**
 *	netif_device_present - is device available or removed
 *	@dev: network device
 *
 * Check if device has not been removed from system.
 */
4072
static inline bool netif_device_present(struct net_device *dev)
L
Linus Torvalds 已提交
4073 4074 4075 4076
{
	return test_bit(__LINK_STATE_PRESENT, &dev->state);
}

4077
void netif_device_detach(struct net_device *dev);
L
Linus Torvalds 已提交
4078

4079
void netif_device_attach(struct net_device *dev);
L
Linus Torvalds 已提交
4080 4081 4082 4083 4084 4085

/*
 * Network interface message level settings
 */

enum {
4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105
	NETIF_MSG_DRV_BIT,
	NETIF_MSG_PROBE_BIT,
	NETIF_MSG_LINK_BIT,
	NETIF_MSG_TIMER_BIT,
	NETIF_MSG_IFDOWN_BIT,
	NETIF_MSG_IFUP_BIT,
	NETIF_MSG_RX_ERR_BIT,
	NETIF_MSG_TX_ERR_BIT,
	NETIF_MSG_TX_QUEUED_BIT,
	NETIF_MSG_INTR_BIT,
	NETIF_MSG_TX_DONE_BIT,
	NETIF_MSG_RX_STATUS_BIT,
	NETIF_MSG_PKTDATA_BIT,
	NETIF_MSG_HW_BIT,
	NETIF_MSG_WOL_BIT,

	/* When you add a new bit above, update netif_msg_class_names array
	 * in net/ethtool/common.c
	 */
	NETIF_MSG_CLASS_COUNT,
L
Linus Torvalds 已提交
4106
};
4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127
/* Both ethtool_ops interface and internal driver implementation use u32 */
static_assert(NETIF_MSG_CLASS_COUNT <= 32);

#define __NETIF_MSG_BIT(bit)	((u32)1 << (bit))
#define __NETIF_MSG(name)	__NETIF_MSG_BIT(NETIF_MSG_ ## name ## _BIT)

#define NETIF_MSG_DRV		__NETIF_MSG(DRV)
#define NETIF_MSG_PROBE		__NETIF_MSG(PROBE)
#define NETIF_MSG_LINK		__NETIF_MSG(LINK)
#define NETIF_MSG_TIMER		__NETIF_MSG(TIMER)
#define NETIF_MSG_IFDOWN	__NETIF_MSG(IFDOWN)
#define NETIF_MSG_IFUP		__NETIF_MSG(IFUP)
#define NETIF_MSG_RX_ERR	__NETIF_MSG(RX_ERR)
#define NETIF_MSG_TX_ERR	__NETIF_MSG(TX_ERR)
#define NETIF_MSG_TX_QUEUED	__NETIF_MSG(TX_QUEUED)
#define NETIF_MSG_INTR		__NETIF_MSG(INTR)
#define NETIF_MSG_TX_DONE	__NETIF_MSG(TX_DONE)
#define NETIF_MSG_RX_STATUS	__NETIF_MSG(RX_STATUS)
#define NETIF_MSG_PKTDATA	__NETIF_MSG(PKTDATA)
#define NETIF_MSG_HW		__NETIF_MSG(HW)
#define NETIF_MSG_WOL		__NETIF_MSG(WOL)
L
Linus Torvalds 已提交
4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152

#define netif_msg_drv(p)	((p)->msg_enable & NETIF_MSG_DRV)
#define netif_msg_probe(p)	((p)->msg_enable & NETIF_MSG_PROBE)
#define netif_msg_link(p)	((p)->msg_enable & NETIF_MSG_LINK)
#define netif_msg_timer(p)	((p)->msg_enable & NETIF_MSG_TIMER)
#define netif_msg_ifdown(p)	((p)->msg_enable & NETIF_MSG_IFDOWN)
#define netif_msg_ifup(p)	((p)->msg_enable & NETIF_MSG_IFUP)
#define netif_msg_rx_err(p)	((p)->msg_enable & NETIF_MSG_RX_ERR)
#define netif_msg_tx_err(p)	((p)->msg_enable & NETIF_MSG_TX_ERR)
#define netif_msg_tx_queued(p)	((p)->msg_enable & NETIF_MSG_TX_QUEUED)
#define netif_msg_intr(p)	((p)->msg_enable & NETIF_MSG_INTR)
#define netif_msg_tx_done(p)	((p)->msg_enable & NETIF_MSG_TX_DONE)
#define netif_msg_rx_status(p)	((p)->msg_enable & NETIF_MSG_RX_STATUS)
#define netif_msg_pktdata(p)	((p)->msg_enable & NETIF_MSG_PKTDATA)
#define netif_msg_hw(p)		((p)->msg_enable & NETIF_MSG_HW)
#define netif_msg_wol(p)	((p)->msg_enable & NETIF_MSG_WOL)

static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
{
	/* use default */
	if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
		return default_msg_enable_bits;
	if (debug_value == 0)	/* no output */
		return 0;
	/* set low N bits */
4153
	return (1U << debug_value) - 1;
L
Linus Torvalds 已提交
4154 4155
}

4156
static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
H
Herbert Xu 已提交
4157
{
4158 4159
	spin_lock(&txq->_xmit_lock);
	txq->xmit_lock_owner = cpu;
4160 4161
}

4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172
static inline bool __netif_tx_acquire(struct netdev_queue *txq)
{
	__acquire(&txq->_xmit_lock);
	return true;
}

static inline void __netif_tx_release(struct netdev_queue *txq)
{
	__release(&txq->_xmit_lock);
}

4173 4174 4175 4176 4177 4178
static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
{
	spin_lock_bh(&txq->_xmit_lock);
	txq->xmit_lock_owner = smp_processor_id();
}

4179
static inline bool __netif_tx_trylock(struct netdev_queue *txq)
4180
{
4181
	bool ok = spin_trylock(&txq->_xmit_lock);
4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198
	if (likely(ok))
		txq->xmit_lock_owner = smp_processor_id();
	return ok;
}

static inline void __netif_tx_unlock(struct netdev_queue *txq)
{
	txq->xmit_lock_owner = -1;
	spin_unlock(&txq->_xmit_lock);
}

static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
{
	txq->xmit_lock_owner = -1;
	spin_unlock_bh(&txq->_xmit_lock);
}

E
Eric Dumazet 已提交
4199 4200 4201 4202 4203 4204
static inline void txq_trans_update(struct netdev_queue *txq)
{
	if (txq->xmit_lock_owner != -1)
		txq->trans_start = jiffies;
}

4205 4206 4207
/* legacy drivers only, netdev_start_xmit() sets txq->trans_start */
static inline void netif_trans_update(struct net_device *dev)
{
F
Florian Westphal 已提交
4208 4209 4210 4211
	struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);

	if (txq->trans_start != jiffies)
		txq->trans_start = jiffies;
4212 4213
}

4214 4215 4216 4217 4218 4219
/**
 *	netif_tx_lock - grab network device transmit lock
 *	@dev: network device
 *
 * Get network device transmit lock
 */
4220 4221
static inline void netif_tx_lock(struct net_device *dev)
{
4222
	unsigned int i;
4223
	int cpu;
4224

4225 4226
	spin_lock(&dev->tx_global_lock);
	cpu = smp_processor_id();
4227 4228
	for (i = 0; i < dev->num_tx_queues; i++) {
		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4229 4230 4231 4232 4233 4234 4235

		/* We are the only thread of execution doing a
		 * freeze, but we have to grab the _xmit_lock in
		 * order to synchronize with threads which are in
		 * the ->hard_start_xmit() handler and already
		 * checked the frozen bit.
		 */
4236
		__netif_tx_lock(txq, cpu);
4237 4238
		set_bit(__QUEUE_STATE_FROZEN, &txq->state);
		__netif_tx_unlock(txq);
4239
	}
H
Herbert Xu 已提交
4240 4241 4242 4243
}

static inline void netif_tx_lock_bh(struct net_device *dev)
{
4244 4245
	local_bh_disable();
	netif_tx_lock(dev);
H
Herbert Xu 已提交
4246 4247 4248 4249
}

static inline void netif_tx_unlock(struct net_device *dev)
{
4250 4251 4252 4253
	unsigned int i;

	for (i = 0; i < dev->num_tx_queues; i++) {
		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4254

4255 4256 4257 4258 4259
		/* No need to grab the _xmit_lock here.  If the
		 * queue is not stopped for another reason, we
		 * force a schedule.
		 */
		clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
4260
		netif_schedule_queue(txq);
4261 4262
	}
	spin_unlock(&dev->tx_global_lock);
H
Herbert Xu 已提交
4263 4264 4265 4266
}

static inline void netif_tx_unlock_bh(struct net_device *dev)
{
4267 4268
	netif_tx_unlock(dev);
	local_bh_enable();
H
Herbert Xu 已提交
4269 4270
}

4271
#define HARD_TX_LOCK(dev, txq, cpu) {			\
4272
	if ((dev->features & NETIF_F_LLTX) == 0) {	\
4273
		__netif_tx_lock(txq, cpu);		\
4274 4275
	} else {					\
		__netif_tx_acquire(txq);		\
4276 4277 4278
	}						\
}

4279 4280 4281
#define HARD_TX_TRYLOCK(dev, txq)			\
	(((dev->features & NETIF_F_LLTX) == 0) ?	\
		__netif_tx_trylock(txq) :		\
4282
		__netif_tx_acquire(txq))
4283

4284
#define HARD_TX_UNLOCK(dev, txq) {			\
4285
	if ((dev->features & NETIF_F_LLTX) == 0) {	\
4286
		__netif_tx_unlock(txq);			\
4287 4288
	} else {					\
		__netif_tx_release(txq);		\
4289 4290 4291
	}						\
}

L
Linus Torvalds 已提交
4292 4293
static inline void netif_tx_disable(struct net_device *dev)
{
4294
	unsigned int i;
4295
	int cpu;
4296

4297 4298
	local_bh_disable();
	cpu = smp_processor_id();
4299 4300
	for (i = 0; i < dev->num_tx_queues; i++) {
		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4301 4302

		__netif_tx_lock(txq, cpu);
4303
		netif_tx_stop_queue(txq);
4304
		__netif_tx_unlock(txq);
4305
	}
4306
	local_bh_enable();
L
Linus Torvalds 已提交
4307 4308
}

4309 4310
static inline void netif_addr_lock(struct net_device *dev)
{
4311
	unsigned char nest_level = 0;
4312

4313 4314 4315 4316
#ifdef CONFIG_LOCKDEP
	nest_level = dev->nested_level;
#endif
	spin_lock_nested(&dev->addr_list_lock, nest_level);
4317 4318
}

4319 4320
static inline void netif_addr_lock_bh(struct net_device *dev)
{
4321 4322 4323 4324 4325 4326 4327
	unsigned char nest_level = 0;

#ifdef CONFIG_LOCKDEP
	nest_level = dev->nested_level;
#endif
	local_bh_disable();
	spin_lock_nested(&dev->addr_list_lock, nest_level);
4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339
}

static inline void netif_addr_unlock(struct net_device *dev)
{
	spin_unlock(&dev->addr_list_lock);
}

static inline void netif_addr_unlock_bh(struct net_device *dev)
{
	spin_unlock_bh(&dev->addr_list_lock);
}

4340
/*
4341
 * dev_addrs walker. Should be used only for read access. Call with
4342 4343 4344
 * rcu_read_lock held.
 */
#define for_each_dev_addr(dev, ha) \
4345
		list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
4346

L
Linus Torvalds 已提交
4347 4348
/* These functions live elsewhere (drivers/net/net_init.c, but related) */

4349
void ether_setup(struct net_device *dev);
L
Linus Torvalds 已提交
4350 4351

/* Support for loadable net-drivers */
4352
struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
4353
				    unsigned char name_assign_type,
4354 4355
				    void (*setup)(struct net_device *),
				    unsigned int txqs, unsigned int rxqs);
4356 4357
#define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
T
Tom Herbert 已提交
4358

4359 4360 4361
#define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
	alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
			 count)
T
Tom Herbert 已提交
4362

4363 4364
int register_netdev(struct net_device *dev);
void unregister_netdev(struct net_device *dev);
4365

4366 4367
int devm_register_netdev(struct device *dev, struct net_device *ndev);

4368
/* General hardware address lists handling functions */
4369 4370 4371 4372
int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
		   struct netdev_hw_addr_list *from_list, int addr_len);
void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
		      struct netdev_hw_addr_list *from_list, int addr_len);
4373 4374 4375 4376 4377
int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
		       struct net_device *dev,
		       int (*sync)(struct net_device *, const unsigned char *),
		       int (*unsync)(struct net_device *,
				     const unsigned char *));
4378 4379 4380 4381 4382 4383 4384 4385 4386 4387
int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
			   struct net_device *dev,
			   int (*sync)(struct net_device *,
				       const unsigned char *, int),
			   int (*unsync)(struct net_device *,
					 const unsigned char *, int));
void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
			      struct net_device *dev,
			      int (*unsync)(struct net_device *,
					    const unsigned char *, int));
4388 4389 4390 4391
void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
			  struct net_device *dev,
			  int (*unsync)(struct net_device *,
					const unsigned char *));
4392
void __hw_addr_init(struct netdev_hw_addr_list *list);
4393

4394
/* Functions used for device addresses handling */
4395 4396 4397 4398 4399 4400
int dev_addr_add(struct net_device *dev, const unsigned char *addr,
		 unsigned char addr_type);
int dev_addr_del(struct net_device *dev, const unsigned char *addr,
		 unsigned char addr_type);
void dev_addr_flush(struct net_device *dev);
int dev_addr_init(struct net_device *dev);
4401 4402

/* Functions used for unicast addresses handling */
4403 4404 4405 4406 4407 4408 4409 4410
int dev_uc_add(struct net_device *dev, const unsigned char *addr);
int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
int dev_uc_del(struct net_device *dev, const unsigned char *addr);
int dev_uc_sync(struct net_device *to, struct net_device *from);
int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
void dev_uc_unsync(struct net_device *to, struct net_device *from);
void dev_uc_flush(struct net_device *dev);
void dev_uc_init(struct net_device *dev);
4411

4412 4413 4414 4415 4416 4417 4418 4419
/**
 *  __dev_uc_sync - Synchonize device's unicast list
 *  @dev:  device to sync
 *  @sync: function to call if address should be added
 *  @unsync: function to call if address should be removed
 *
 *  Add newly added addresses to the interface, and release
 *  addresses that have been deleted.
B
Bjorn Helgaas 已提交
4420
 */
4421 4422 4423 4424 4425 4426 4427 4428 4429 4430
static inline int __dev_uc_sync(struct net_device *dev,
				int (*sync)(struct net_device *,
					    const unsigned char *),
				int (*unsync)(struct net_device *,
					      const unsigned char *))
{
	return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
}

/**
4431
 *  __dev_uc_unsync - Remove synchronized addresses from device
4432 4433 4434 4435
 *  @dev:  device to sync
 *  @unsync: function to call if address should be removed
 *
 *  Remove all addresses that were added to the device by dev_uc_sync().
B
Bjorn Helgaas 已提交
4436
 */
4437 4438 4439 4440 4441 4442 4443
static inline void __dev_uc_unsync(struct net_device *dev,
				   int (*unsync)(struct net_device *,
						 const unsigned char *))
{
	__hw_addr_unsync_dev(&dev->uc, dev, unsync);
}

4444
/* Functions used for multicast addresses handling */
4445 4446 4447 4448 4449 4450 4451 4452 4453 4454
int dev_mc_add(struct net_device *dev, const unsigned char *addr);
int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
int dev_mc_del(struct net_device *dev, const unsigned char *addr);
int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
int dev_mc_sync(struct net_device *to, struct net_device *from);
int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
void dev_mc_unsync(struct net_device *to, struct net_device *from);
void dev_mc_flush(struct net_device *dev);
void dev_mc_init(struct net_device *dev);
4455

4456 4457 4458 4459 4460 4461 4462 4463
/**
 *  __dev_mc_sync - Synchonize device's multicast list
 *  @dev:  device to sync
 *  @sync: function to call if address should be added
 *  @unsync: function to call if address should be removed
 *
 *  Add newly added addresses to the interface, and release
 *  addresses that have been deleted.
B
Bjorn Helgaas 已提交
4464
 */
4465 4466 4467 4468 4469 4470 4471 4472 4473 4474
static inline int __dev_mc_sync(struct net_device *dev,
				int (*sync)(struct net_device *,
					    const unsigned char *),
				int (*unsync)(struct net_device *,
					      const unsigned char *))
{
	return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
}

/**
4475
 *  __dev_mc_unsync - Remove synchronized addresses from device
4476 4477 4478 4479
 *  @dev:  device to sync
 *  @unsync: function to call if address should be removed
 *
 *  Remove all addresses that were added to the device by dev_mc_sync().
B
Bjorn Helgaas 已提交
4480
 */
4481 4482 4483 4484 4485 4486 4487
static inline void __dev_mc_unsync(struct net_device *dev,
				   int (*unsync)(struct net_device *,
						 const unsigned char *))
{
	__hw_addr_unsync_dev(&dev->mc, dev, unsync);
}

4488
/* Functions used for secondary unicast and multicast support */
4489 4490 4491 4492 4493 4494 4495
void dev_set_rx_mode(struct net_device *dev);
void __dev_set_rx_mode(struct net_device *dev);
int dev_set_promiscuity(struct net_device *dev, int inc);
int dev_set_allmulti(struct net_device *dev, int inc);
void netdev_state_change(struct net_device *dev);
void netdev_notify_peers(struct net_device *dev);
void netdev_features_change(struct net_device *dev);
L
Linus Torvalds 已提交
4496
/* Load a device via the kmod */
4497 4498 4499 4500 4501
void dev_load(struct net *net, const char *name);
struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
					struct rtnl_link_stats64 *storage);
void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
			     const struct net_device_stats *netdev_stats);
4502 4503
void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
			   const struct pcpu_sw_netstats __percpu *netstats);
4504

L
Linus Torvalds 已提交
4505
extern int		netdev_max_backlog;
E
Eric Dumazet 已提交
4506
extern int		netdev_tstamp_prequeue;
L
Linus Torvalds 已提交
4507
extern int		weight_p;
4508 4509 4510 4511
extern int		dev_weight_rx_bias;
extern int		dev_weight_tx_bias;
extern int		dev_rx_weight;
extern int		dev_tx_weight;
4512
extern int		gro_normal_batch;
J
Jiri Pirko 已提交
4513

4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524
enum {
	NESTED_SYNC_IMM_BIT,
	NESTED_SYNC_TODO_BIT,
};

#define __NESTED_SYNC_BIT(bit)	((u32)1 << (bit))
#define __NESTED_SYNC(name)	__NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)

#define NESTED_SYNC_IMM		__NESTED_SYNC(IMM)
#define NESTED_SYNC_TODO	__NESTED_SYNC(TODO)

4525
struct netdev_nested_priv {
4526
	unsigned char flags;
4527 4528 4529
	void *data;
};

4530
bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
4531 4532
struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
						     struct list_head **iter);
4533 4534
struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
						     struct list_head **iter);
4535

4536 4537 4538 4539 4540 4541 4542 4543 4544 4545
#ifdef CONFIG_LOCKDEP
static LIST_HEAD(net_unlink_list);

static inline void net_unlink_todo(struct net_device *dev)
{
	if (list_empty(&dev->unlink_list))
		list_add_tail(&dev->unlink_list, &net_unlink_list);
}
#endif

4546 4547 4548 4549 4550 4551 4552
/* iterate through upper list, must be called under RCU read lock */
#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
	for (iter = &(dev)->adj_list.upper, \
	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
	     updev; \
	     updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))

4553 4554
int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
				  int (*fn)(struct net_device *upper_dev,
4555 4556
					    struct netdev_nested_priv *priv),
				  struct netdev_nested_priv *priv);
4557 4558 4559 4560

bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
				  struct net_device *upper_dev);

4561 4562
bool netdev_has_any_upper_dev(struct net_device *dev);

4563 4564 4565 4566
void *netdev_lower_get_next_private(struct net_device *dev,
				    struct list_head **iter);
void *netdev_lower_get_next_private_rcu(struct net_device *dev,
					struct list_head **iter);
4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579

#define netdev_for_each_lower_private(dev, priv, iter) \
	for (iter = (dev)->adj_list.lower.next, \
	     priv = netdev_lower_get_next_private(dev, &(iter)); \
	     priv; \
	     priv = netdev_lower_get_next_private(dev, &(iter)))

#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
	for (iter = &(dev)->adj_list.lower, \
	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
	     priv; \
	     priv = netdev_lower_get_next_private_rcu(dev, &(iter)))

4580 4581
void *netdev_lower_get_next(struct net_device *dev,
				struct list_head **iter);
4582

4583
#define netdev_for_each_lower_dev(dev, ldev, iter) \
4584
	for (iter = (dev)->adj_list.lower.next, \
4585 4586 4587 4588
	     ldev = netdev_lower_get_next(dev, &(iter)); \
	     ldev; \
	     ldev = netdev_lower_get_next(dev, &(iter)))

4589
struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
4590
					     struct list_head **iter);
4591 4592
int netdev_walk_all_lower_dev(struct net_device *dev,
			      int (*fn)(struct net_device *lower_dev,
4593 4594
					struct netdev_nested_priv *priv),
			      struct netdev_nested_priv *priv);
4595 4596
int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
				  int (*fn)(struct net_device *lower_dev,
4597 4598
					    struct netdev_nested_priv *priv),
				  struct netdev_nested_priv *priv);
4599

4600
void *netdev_adjacent_get_private(struct list_head *adj_list);
4601
void *netdev_lower_get_first_private_rcu(struct net_device *dev);
4602 4603
struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
4604 4605
int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
			  struct netlink_ext_ack *extack);
4606
int netdev_master_upper_dev_link(struct net_device *dev,
4607
				 struct net_device *upper_dev,
4608 4609
				 void *upper_priv, void *upper_info,
				 struct netlink_ext_ack *extack);
4610 4611
void netdev_upper_dev_unlink(struct net_device *dev,
			     struct net_device *upper_dev);
4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
int netdev_adjacent_change_prepare(struct net_device *old_dev,
				   struct net_device *new_dev,
				   struct net_device *dev,
				   struct netlink_ext_ack *extack);
void netdev_adjacent_change_commit(struct net_device *old_dev,
				   struct net_device *new_dev,
				   struct net_device *dev);
void netdev_adjacent_change_abort(struct net_device *old_dev,
				  struct net_device *new_dev,
				  struct net_device *dev);
4622
void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
4623 4624
void *netdev_lower_dev_get_private(struct net_device *dev,
				   struct net_device *lower_dev);
4625 4626
void netdev_lower_state_changed(struct net_device *lower_dev,
				void *lower_state_info);
4627 4628 4629

/* RSS keys are 40 or 52 bytes long */
#define NETDEV_RSS_KEY_LEN 52
4630
extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
4631 4632
void netdev_rss_key_fill(void *buffer, size_t len);

4633
int skb_checksum_help(struct sk_buff *skb);
4634
int skb_crc32c_csum_help(struct sk_buff *skb);
4635 4636 4637
int skb_csum_hwoffload_help(struct sk_buff *skb,
			    const netdev_features_t features);

4638 4639 4640 4641
struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
				  netdev_features_t features, bool tx_path);
struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
				    netdev_features_t features);
4642

4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655
struct netdev_bonding_info {
	ifslave	slave;
	ifbond	master;
};

struct netdev_notifier_bonding_info {
	struct netdev_notifier_info info; /* must be first */
	struct netdev_bonding_info  bonding_info;
};

void netdev_bonding_info_change(struct net_device *dev,
				struct netdev_bonding_info *bonding_info);

4656 4657 4658 4659 4660 4661 4662 4663 4664
#if IS_ENABLED(CONFIG_ETHTOOL_NETLINK)
void ethtool_notify(struct net_device *dev, unsigned int cmd, const void *data);
#else
static inline void ethtool_notify(struct net_device *dev, unsigned int cmd,
				  const void *data)
{
}
#endif

4665 4666 4667 4668 4669
static inline
struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
{
	return __skb_gso_segment(skb, features, true);
}
4670
__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
4671 4672 4673 4674

static inline bool can_checksum_protocol(netdev_features_t features,
					 __be16 protocol)
{
4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692
	if (protocol == htons(ETH_P_FCOE))
		return !!(features & NETIF_F_FCOE_CRC);

	/* Assume this is an IP checksum (not SCTP CRC) */

	if (features & NETIF_F_HW_CSUM) {
		/* Can checksum everything */
		return true;
	}

	switch (protocol) {
	case htons(ETH_P_IP):
		return !!(features & NETIF_F_IP_CSUM);
	case htons(ETH_P_IPV6):
		return !!(features & NETIF_F_IPV6_CSUM);
	default:
		return false;
	}
4693
}
4694

4695
#ifdef CONFIG_BUG
4696
void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb);
4697
#else
4698 4699
static inline void netdev_rx_csum_fault(struct net_device *dev,
					struct sk_buff *skb)
4700 4701 4702
{
}
#endif
L
Linus Torvalds 已提交
4703
/* rx skb timestamps */
4704 4705
void net_enable_timestamp(void);
void net_disable_timestamp(void);
L
Linus Torvalds 已提交
4706

4707
#ifdef CONFIG_PROC_FS
4708
int __init dev_proc_init(void);
4709 4710
#else
#define dev_proc_init() 0
4711 4712
#endif

D
David S. Miller 已提交
4713
static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
4714 4715
					      struct sk_buff *skb, struct net_device *dev,
					      bool more)
D
David S. Miller 已提交
4716
{
4717
	__this_cpu_write(softnet_data.xmit.more, more);
4718
	return ops->ndo_start_xmit(skb, dev);
D
David S. Miller 已提交
4719 4720
}

4721 4722 4723 4724 4725
static inline bool netdev_xmit_more(void)
{
	return __this_cpu_read(softnet_data.xmit.more);
}

4726
static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
4727
					    struct netdev_queue *txq, bool more)
D
David S. Miller 已提交
4728 4729
{
	const struct net_device_ops *ops = dev->netdev_ops;
4730
	netdev_tx_t rc;
D
David S. Miller 已提交
4731

4732
	rc = __netdev_start_xmit(ops, skb, dev, more);
4733 4734 4735 4736
	if (rc == NETDEV_TX_OK)
		txq_trans_update(txq);

	return rc;
D
David S. Miller 已提交
4737 4738
}

4739
int netdev_class_create_file_ns(const struct class_attribute *class_attr,
4740
				const void *ns);
4741
void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
4742
				 const void *ns);
4743

4744
extern const struct kobj_ns_type_operations net_ns_type_operations;
4745

4746
const char *netdev_drivername(const struct net_device *dev);
4747

4748
void linkwatch_run_queue(void);
4749

4750 4751 4752
static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
							  netdev_features_t f2)
{
4753 4754
	if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
		if (f1 & NETIF_F_HW_CSUM)
4755
			f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4756
		else
4757
			f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4758
	}
4759

4760
	return f1 & f2;
4761 4762
}

4763 4764
static inline netdev_features_t netdev_get_wanted_features(
	struct net_device *dev)
4765 4766 4767
{
	return (dev->features & ~dev->hw_features) | dev->wanted_features;
}
4768 4769
netdev_features_t netdev_increment_features(netdev_features_t all,
	netdev_features_t one, netdev_features_t mask);
4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780

/* Allow TSO being used on stacked device :
 * Performing the GSO segmentation before last device
 * is a performance improvement.
 */
static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
							netdev_features_t mask)
{
	return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
}

4781
int __netdev_update_features(struct net_device *dev);
4782
void netdev_update_features(struct net_device *dev);
4783
void netdev_change_features(struct net_device *dev);
4784

4785 4786 4787
void netif_stacked_transfer_operstate(const struct net_device *rootdev,
					struct net_device *dev);

4788 4789 4790
netdev_features_t passthru_features_check(struct sk_buff *skb,
					  struct net_device *dev,
					  netdev_features_t features);
4791
netdev_features_t netif_skb_features(struct sk_buff *skb);
4792

4793
static inline bool net_gso_ok(netdev_features_t features, int gso_type)
4794
{
4795
	netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
4796 4797 4798 4799 4800

	/* check flags correspondence */
	BUILD_BUG_ON(SKB_GSO_TCPV4   != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
	BUILD_BUG_ON(SKB_GSO_DODGY   != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
	BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
4801
	BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
4802 4803
	BUILD_BUG_ON(SKB_GSO_TCPV6   != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
	BUILD_BUG_ON(SKB_GSO_FCOE    != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
4804 4805
	BUILD_BUG_ON(SKB_GSO_GRE     != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
	BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
4806 4807
	BUILD_BUG_ON(SKB_GSO_IPXIP4  != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
	BUILD_BUG_ON(SKB_GSO_IPXIP6  != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
4808 4809
	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
	BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
4810
	BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
4811
	BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
M
Marcelo Ricardo Leitner 已提交
4812
	BUILD_BUG_ON(SKB_GSO_SCTP    != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
S
Steffen Klassert 已提交
4813
	BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
4814
	BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
4815
	BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
4816
	BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
4817

4818
	return (features & feature) == feature;
4819 4820
}

4821
static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
4822
{
H
Herbert Xu 已提交
4823
	return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
4824
	       (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
4825 4826
}

4827
static inline bool netif_needs_gso(struct sk_buff *skb,
4828
				   netdev_features_t features)
4829
{
4830
	return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
4831 4832
		unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
			 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
4833 4834
}

4835 4836 4837 4838 4839 4840
static inline void netif_set_gso_max_size(struct net_device *dev,
					  unsigned int size)
{
	dev->gso_max_size = size;
}

4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853
static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
					int pulled_hlen, u16 mac_offset,
					int mac_len)
{
	skb->protocol = protocol;
	skb->encapsulation = 1;
	skb_push(skb, pulled_hlen);
	skb_reset_transport_header(skb);
	skb->mac_header = mac_offset;
	skb->network_header = skb->mac_header + mac_len;
	skb->mac_len = mac_len;
}

4854 4855 4856 4857 4858
static inline bool netif_is_macsec(const struct net_device *dev)
{
	return dev->priv_flags & IFF_MACSEC;
}

4859
static inline bool netif_is_macvlan(const struct net_device *dev)
4860 4861 4862 4863
{
	return dev->priv_flags & IFF_MACVLAN;
}

4864
static inline bool netif_is_macvlan_port(const struct net_device *dev)
4865 4866 4867 4868
{
	return dev->priv_flags & IFF_MACVLAN_PORT;
}

4869
static inline bool netif_is_bond_master(const struct net_device *dev)
4870 4871 4872 4873
{
	return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
}

4874
static inline bool netif_is_bond_slave(const struct net_device *dev)
J
Jiri Pirko 已提交
4875 4876 4877 4878
{
	return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
}

4879 4880 4881 4882 4883
static inline bool netif_supports_nofcs(struct net_device *dev)
{
	return dev->priv_flags & IFF_SUPP_NOFCS;
}

4884 4885 4886 4887 4888
static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
{
	return dev->priv_flags & IFF_L3MDEV_RX_HANDLER;
}

4889
static inline bool netif_is_l3_master(const struct net_device *dev)
4890
{
4891
	return dev->priv_flags & IFF_L3MDEV_MASTER;
4892 4893
}

D
David Ahern 已提交
4894 4895 4896 4897 4898
static inline bool netif_is_l3_slave(const struct net_device *dev)
{
	return dev->priv_flags & IFF_L3MDEV_SLAVE;
}

4899 4900 4901 4902 4903
static inline bool netif_is_bridge_master(const struct net_device *dev)
{
	return dev->priv_flags & IFF_EBRIDGE;
}

4904 4905 4906 4907 4908
static inline bool netif_is_bridge_port(const struct net_device *dev)
{
	return dev->priv_flags & IFF_BRIDGE_PORT;
}

4909 4910 4911 4912 4913
static inline bool netif_is_ovs_master(const struct net_device *dev)
{
	return dev->priv_flags & IFF_OPENVSWITCH;
}

J
Jiri Pirko 已提交
4914 4915 4916 4917 4918
static inline bool netif_is_ovs_port(const struct net_device *dev)
{
	return dev->priv_flags & IFF_OVS_DATAPATH;
}

4919 4920 4921 4922 4923
static inline bool netif_is_any_bridge_port(const struct net_device *dev)
{
	return netif_is_bridge_port(dev) || netif_is_ovs_port(dev);
}

4924
static inline bool netif_is_team_master(const struct net_device *dev)
J
Jiri Pirko 已提交
4925 4926 4927 4928
{
	return dev->priv_flags & IFF_TEAM;
}

4929
static inline bool netif_is_team_port(const struct net_device *dev)
J
Jiri Pirko 已提交
4930 4931 4932 4933
{
	return dev->priv_flags & IFF_TEAM_PORT;
}

4934
static inline bool netif_is_lag_master(const struct net_device *dev)
J
Jiri Pirko 已提交
4935 4936 4937 4938
{
	return netif_is_bond_master(dev) || netif_is_team_master(dev);
}

4939
static inline bool netif_is_lag_port(const struct net_device *dev)
J
Jiri Pirko 已提交
4940 4941 4942 4943
{
	return netif_is_bond_slave(dev) || netif_is_team_port(dev);
}

4944 4945 4946 4947 4948
static inline bool netif_is_rxfh_configured(const struct net_device *dev)
{
	return dev->priv_flags & IFF_RXFH_CONFIGURED;
}

4949 4950 4951 4952 4953 4954 4955 4956 4957 4958
static inline bool netif_is_failover(const struct net_device *dev)
{
	return dev->priv_flags & IFF_FAILOVER;
}

static inline bool netif_is_failover_slave(const struct net_device *dev)
{
	return dev->priv_flags & IFF_FAILOVER_SLAVE;
}

4959 4960 4961 4962 4963 4964
/* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
static inline void netif_keep_dst(struct net_device *dev)
{
	dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
}

4965 4966 4967 4968 4969 4970 4971
/* return true if dev can't cope with mtu frames that need vlan tag insertion */
static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
{
	/* TODO: reserve and use an additional IFF bit, if we get more users */
	return dev->priv_flags & IFF_MACSEC;
}

4972
extern struct pernet_operations __net_initdata loopback_net_ops;
4973

4974 4975 4976 4977 4978 4979
/* Logging, debugging and troubleshooting/diagnostic helpers. */

/* netdev_printk helpers, similar to dev_printk */

static inline const char *netdev_name(const struct net_device *dev)
{
4980 4981
	if (!dev->name[0] || strchr(dev->name, '%'))
		return "(unnamed net_device)";
4982 4983 4984
	return dev->name;
}

4985 4986 4987 4988 4989
static inline bool netdev_unregistering(const struct net_device *dev)
{
	return dev->reg_state == NETREG_UNREGISTERING;
}

4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004
static inline const char *netdev_reg_state(const struct net_device *dev)
{
	switch (dev->reg_state) {
	case NETREG_UNINITIALIZED: return " (uninitialized)";
	case NETREG_REGISTERED: return "";
	case NETREG_UNREGISTERING: return " (unregistering)";
	case NETREG_UNREGISTERED: return " (unregistered)";
	case NETREG_RELEASED: return " (released)";
	case NETREG_DUMMY: return " (dummy)";
	}

	WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
	return " (unknown)";
}

5005
__printf(3, 4) __cold
5006 5007
void netdev_printk(const char *level, const struct net_device *dev,
		   const char *format, ...);
5008
__printf(2, 3) __cold
5009
void netdev_emerg(const struct net_device *dev, const char *format, ...);
5010
__printf(2, 3) __cold
5011
void netdev_alert(const struct net_device *dev, const char *format, ...);
5012
__printf(2, 3) __cold
5013
void netdev_crit(const struct net_device *dev, const char *format, ...);
5014
__printf(2, 3) __cold
5015
void netdev_err(const struct net_device *dev, const char *format, ...);
5016
__printf(2, 3) __cold
5017
void netdev_warn(const struct net_device *dev, const char *format, ...);
5018
__printf(2, 3) __cold
5019
void netdev_notice(const struct net_device *dev, const char *format, ...);
5020
__printf(2, 3) __cold
5021
void netdev_info(const struct net_device *dev, const char *format, ...);
5022

5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047
#define netdev_level_once(level, dev, fmt, ...)			\
do {								\
	static bool __print_once __read_mostly;			\
								\
	if (!__print_once) {					\
		__print_once = true;				\
		netdev_printk(level, dev, fmt, ##__VA_ARGS__);	\
	}							\
} while (0)

#define netdev_emerg_once(dev, fmt, ...) \
	netdev_level_once(KERN_EMERG, dev, fmt, ##__VA_ARGS__)
#define netdev_alert_once(dev, fmt, ...) \
	netdev_level_once(KERN_ALERT, dev, fmt, ##__VA_ARGS__)
#define netdev_crit_once(dev, fmt, ...) \
	netdev_level_once(KERN_CRIT, dev, fmt, ##__VA_ARGS__)
#define netdev_err_once(dev, fmt, ...) \
	netdev_level_once(KERN_ERR, dev, fmt, ##__VA_ARGS__)
#define netdev_warn_once(dev, fmt, ...) \
	netdev_level_once(KERN_WARNING, dev, fmt, ##__VA_ARGS__)
#define netdev_notice_once(dev, fmt, ...) \
	netdev_level_once(KERN_NOTICE, dev, fmt, ##__VA_ARGS__)
#define netdev_info_once(dev, fmt, ...) \
	netdev_level_once(KERN_INFO, dev, fmt, ##__VA_ARGS__)

5048 5049 5050
#define MODULE_ALIAS_NETDEV(device) \
	MODULE_ALIAS("netdev-" device)

5051 5052
#if defined(CONFIG_DYNAMIC_DEBUG) || \
	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
5053 5054
#define netdev_dbg(__dev, format, args...)			\
do {								\
5055
	dynamic_netdev_dbg(__dev, format, ##args);		\
5056
} while (0)
5057 5058 5059
#elif defined(DEBUG)
#define netdev_dbg(__dev, format, args...)			\
	netdev_printk(KERN_DEBUG, __dev, format, ##args)
5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085
#else
#define netdev_dbg(__dev, format, args...)			\
({								\
	if (0)							\
		netdev_printk(KERN_DEBUG, __dev, format, ##args); \
})
#endif

#if defined(VERBOSE_DEBUG)
#define netdev_vdbg	netdev_dbg
#else

#define netdev_vdbg(dev, format, args...)			\
({								\
	if (0)							\
		netdev_printk(KERN_DEBUG, dev, format, ##args);	\
	0;							\
})
#endif

/*
 * netdev_WARN() acts like dev_printk(), but with the key difference
 * of using a WARN/WARN_ON to get the message out, including the
 * file/line information and a backtrace.
 */
#define netdev_WARN(dev, format, args...)			\
5086
	WARN(1, "netdevice: %s%s: " format, netdev_name(dev),	\
5087
	     netdev_reg_state(dev), ##args)
5088

G
Gal Pressman 已提交
5089
#define netdev_WARN_ONCE(dev, format, args...)				\
5090
	WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev),	\
5091 5092
		  netdev_reg_state(dev), ##args)

5093 5094 5095 5096 5097 5098 5099 5100
/* netif printk helpers, similar to netdev_printk */

#define netif_printk(priv, type, level, dev, fmt, args...)	\
do {					  			\
	if (netif_msg_##type(priv))				\
		netdev_printk(level, (dev), fmt, ##args);	\
} while (0)

5101 5102 5103 5104 5105 5106
#define netif_level(level, priv, type, dev, fmt, args...)	\
do {								\
	if (netif_msg_##type(priv))				\
		netdev_##level(dev, fmt, ##args);		\
} while (0)

5107
#define netif_emerg(priv, type, dev, fmt, args...)		\
5108
	netif_level(emerg, priv, type, dev, fmt, ##args)
5109
#define netif_alert(priv, type, dev, fmt, args...)		\
5110
	netif_level(alert, priv, type, dev, fmt, ##args)
5111
#define netif_crit(priv, type, dev, fmt, args...)		\
5112
	netif_level(crit, priv, type, dev, fmt, ##args)
5113
#define netif_err(priv, type, dev, fmt, args...)		\
5114
	netif_level(err, priv, type, dev, fmt, ##args)
5115
#define netif_warn(priv, type, dev, fmt, args...)		\
5116
	netif_level(warn, priv, type, dev, fmt, ##args)
5117
#define netif_notice(priv, type, dev, fmt, args...)		\
5118
	netif_level(notice, priv, type, dev, fmt, ##args)
5119
#define netif_info(priv, type, dev, fmt, args...)		\
5120
	netif_level(info, priv, type, dev, fmt, ##args)
5121

5122 5123
#if defined(CONFIG_DYNAMIC_DEBUG) || \
	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
5124 5125 5126
#define netif_dbg(priv, type, netdev, format, args...)		\
do {								\
	if (netif_msg_##type(priv))				\
5127
		dynamic_netdev_dbg(netdev, format, ##args);	\
5128
} while (0)
5129 5130 5131
#elif defined(DEBUG)
#define netif_dbg(priv, type, dev, format, args...)		\
	netif_printk(priv, type, KERN_DEBUG, dev, format, ##args)
5132 5133 5134 5135 5136 5137 5138 5139 5140
#else
#define netif_dbg(priv, type, dev, format, args...)			\
({									\
	if (0)								\
		netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
	0;								\
})
#endif

5141 5142 5143 5144 5145 5146 5147 5148 5149
/* if @cond then downgrade to debug, else print at @level */
#define netif_cond_dbg(priv, type, netdev, cond, level, fmt, args...)     \
	do {                                                              \
		if (cond)                                                 \
			netif_dbg(priv, type, netdev, fmt, ##args);       \
		else                                                      \
			netif_ ## level(priv, type, netdev, fmt, ##args); \
	} while (0)

5150
#if defined(VERBOSE_DEBUG)
5151
#define netif_vdbg	netif_dbg
5152 5153 5154 5155
#else
#define netif_vdbg(priv, type, dev, format, args...)		\
({								\
	if (0)							\
5156
		netif_printk(priv, type, KERN_DEBUG, dev, format, ##args); \
5157 5158 5159
	0;							\
})
#endif
5160

5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182
/*
 *	The list of packet types we will receive (as opposed to discard)
 *	and the routines to invoke.
 *
 *	Why 16. Because with 16 the only overlap we get on a hash of the
 *	low nibble of the protocol value is RARP/SNAP/X.25.
 *
 *		0800	IP
 *		0001	802.3
 *		0002	AX.25
 *		0004	802.2
 *		8035	RARP
 *		0005	SNAP
 *		0805	X.25
 *		0806	ARP
 *		8137	IPX
 *		0009	Localtalk
 *		86DD	IPv6
 */
#define PTYPE_HASH_SIZE	(16)
#define PTYPE_HASH_MASK	(PTYPE_HASH_SIZE - 1)

5183 5184
extern struct net_device *blackhole_netdev;

5185
#endif	/* _LINUX_NETDEVICE_H */