dev.c 156.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9
/*
 * 	NET3	Protocol independent device support routines.
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 *
 *	Derived from the non IP parts of dev.c 1.0.19
10
 * 		Authors:	Ross Biro
L
Linus Torvalds 已提交
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
 *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *				Mark Evans, <evansmp@uhura.aston.ac.uk>
 *
 *	Additional Authors:
 *		Florian la Roche <rzsfl@rz.uni-sb.de>
 *		Alan Cox <gw4pts@gw4pts.ampr.org>
 *		David Hinds <dahinds@users.sourceforge.net>
 *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 *		Adam Sulmicki <adam@cfar.umd.edu>
 *              Pekka Riikonen <priikone@poesidon.pspt.fi>
 *
 *	Changes:
 *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
 *              			to 2 if register_netdev gets called
 *              			before net_dev_init & also removed a
 *              			few lines of code in the process.
 *		Alan Cox	:	device private ioctl copies fields back.
 *		Alan Cox	:	Transmit queue code does relevant
 *					stunts to keep the queue safe.
 *		Alan Cox	:	Fixed double lock.
 *		Alan Cox	:	Fixed promisc NULL pointer trap
 *		????????	:	Support the full private ioctl range
 *		Alan Cox	:	Moved ioctl permission check into
 *					drivers
 *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
 *		Alan Cox	:	100 backlog just doesn't cut it when
 *					you start doing multicast video 8)
 *		Alan Cox	:	Rewrote net_bh and list manager.
 *		Alan Cox	: 	Fix ETH_P_ALL echoback lengths.
 *		Alan Cox	:	Took out transmit every packet pass
 *					Saved a few bytes in the ioctl handler
 *		Alan Cox	:	Network driver sets packet type before
 *					calling netif_rx. Saves a function
 *					call a packet.
 *		Alan Cox	:	Hashed net_bh()
 *		Richard Kooijman:	Timestamp fixes.
 *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
 *		Alan Cox	:	Device lock protection.
 *		Alan Cox	: 	Fixed nasty side effect of device close
 *					changes.
 *		Rudi Cilibrasi	:	Pass the right thing to
 *					set_mac_address()
 *		Dave Miller	:	32bit quantity for the device lock to
 *					make it work out on a Sparc.
 *		Bjorn Ekwall	:	Added KERNELD hack.
 *		Alan Cox	:	Cleaned up the backlog initialise.
 *		Craig Metz	:	SIOCGIFCONF fix if space for under
 *					1 device.
 *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
 *					is no device open function.
 *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
 *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
 *		Cyrus Durgin	:	Cleaned for KMOD
 *		Adam Sulmicki   :	Bug Fix : Network Device Unload
 *					A network device unload needs to purge
 *					the backlog queue.
 *	Paul Rusty Russell	:	SIOCSIFNAME
 *              Pekka Riikonen  :	Netdev boot-time settings code
 *              Andrew Morton   :       Make unregister_netdevice wait
 *              			indefinitely on dev->refcnt
 * 		J Hadi Salim	:	- Backlog queue sampling
 *				        - netif_rx() feedback
 */

#include <asm/uaccess.h>
#include <linux/bitops.h>
77
#include <linux/capability.h>
L
Linus Torvalds 已提交
78 79 80
#include <linux/cpu.h>
#include <linux/types.h>
#include <linux/kernel.h>
81
#include <linux/hash.h>
82
#include <linux/slab.h>
L
Linus Torvalds 已提交
83
#include <linux/sched.h>
A
Arjan van de Ven 已提交
84
#include <linux/mutex.h>
L
Linus Torvalds 已提交
85 86 87 88 89 90 91 92 93
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/if_ether.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
94
#include <linux/ethtool.h>
L
Linus Torvalds 已提交
95 96
#include <linux/notifier.h>
#include <linux/skbuff.h>
97
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
98 99 100 101 102 103
#include <net/sock.h>
#include <linux/rtnetlink.h>
#include <linux/stat.h>
#include <net/dst.h>
#include <net/pkt_sched.h>
#include <net/checksum.h>
104
#include <net/xfrm.h>
L
Linus Torvalds 已提交
105 106 107 108 109 110 111 112
#include <linux/highmem.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netpoll.h>
#include <linux/rcupdate.h>
#include <linux/delay.h>
#include <net/iw_handler.h>
#include <asm/current.h>
S
Steve Grubb 已提交
113
#include <linux/audit.h>
114
#include <linux/dmaengine.h>
115
#include <linux/err.h>
116
#include <linux/ctype.h>
117
#include <linux/if_arp.h>
118
#include <linux/if_vlan.h>
119
#include <linux/ip.h>
120
#include <net/ip.h>
121 122
#include <linux/ipv6.h>
#include <linux/in.h>
D
David S. Miller 已提交
123 124
#include <linux/jhash.h>
#include <linux/random.h>
125
#include <trace/events/napi.h>
126
#include <trace/events/net.h>
127
#include <trace/events/skb.h>
128
#include <linux/pci.h>
129
#include <linux/inetdevice.h>
130
#include <linux/cpu_rmap.h>
131
#include <linux/static_key.h>
L
Linus Torvalds 已提交
132

133 134
#include "net-sysfs.h"

135 136 137
/* Instead of increasing this, you should create a hash table. */
#define MAX_GRO_SKBS 8

H
Herbert Xu 已提交
138 139 140
/* This should be increased if a protocol with a bigger head is added. */
#define GRO_MAX_HEAD (MAX_HEADER + 128)

L
Linus Torvalds 已提交
141
static DEFINE_SPINLOCK(ptype_lock);
142
static DEFINE_SPINLOCK(offload_lock);
143 144
struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
struct list_head ptype_all __read_mostly;	/* Taps */
145
static struct list_head offload_base __read_mostly;
L
Linus Torvalds 已提交
146 147

/*
148
 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
L
Linus Torvalds 已提交
149 150
 * semaphore.
 *
151
 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
L
Linus Torvalds 已提交
152 153
 *
 * Writers must hold the rtnl semaphore while they loop through the
154
 * dev_base_head list, and hold dev_base_lock for writing when they do the
L
Linus Torvalds 已提交
155 156 157 158 159 160 161 162 163 164 165 166 167 168
 * actual updates.  This allows pure readers to access the list even
 * while a writer is preparing to update it.
 *
 * To put it another way, dev_base_lock is held for writing only to
 * protect against pure readers; the rtnl semaphore provides the
 * protection against other writers.
 *
 * See, for example usages, register_netdevice() and
 * unregister_netdevice(), which must be called with the rtnl
 * semaphore held.
 */
DEFINE_RWLOCK(dev_base_lock);
EXPORT_SYMBOL(dev_base_lock);

169
seqcount_t devnet_rename_seq;
170

171 172 173 174 175
static inline void dev_base_seq_inc(struct net *net)
{
	while (++net->dev_base_seq == 0);
}

176
static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
L
Linus Torvalds 已提交
177
{
178 179
	unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));

180
	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
L
Linus Torvalds 已提交
181 182
}

183
static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
L
Linus Torvalds 已提交
184
{
185
	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
L
Linus Torvalds 已提交
186 187
}

E
Eric Dumazet 已提交
188
static inline void rps_lock(struct softnet_data *sd)
189 190
{
#ifdef CONFIG_RPS
E
Eric Dumazet 已提交
191
	spin_lock(&sd->input_pkt_queue.lock);
192 193 194
#endif
}

E
Eric Dumazet 已提交
195
static inline void rps_unlock(struct softnet_data *sd)
196 197
{
#ifdef CONFIG_RPS
E
Eric Dumazet 已提交
198
	spin_unlock(&sd->input_pkt_queue.lock);
199 200 201
#endif
}

202
/* Device list insertion */
203
static void list_netdevice(struct net_device *dev)
204
{
205
	struct net *net = dev_net(dev);
206 207 208 209

	ASSERT_RTNL();

	write_lock_bh(&dev_base_lock);
210
	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
211
	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
212 213
	hlist_add_head_rcu(&dev->index_hlist,
			   dev_index_hash(net, dev->ifindex));
214
	write_unlock_bh(&dev_base_lock);
215 216

	dev_base_seq_inc(net);
217 218
}

219 220 221
/* Device list removal
 * caller must respect a RCU grace period before freeing/reusing dev
 */
222 223 224 225 226 227
static void unlist_netdevice(struct net_device *dev)
{
	ASSERT_RTNL();

	/* Unlink dev from the device chain */
	write_lock_bh(&dev_base_lock);
228
	list_del_rcu(&dev->dev_list);
229
	hlist_del_rcu(&dev->name_hlist);
230
	hlist_del_rcu(&dev->index_hlist);
231
	write_unlock_bh(&dev_base_lock);
232 233

	dev_base_seq_inc(dev_net(dev));
234 235
}

L
Linus Torvalds 已提交
236 237 238 239
/*
 *	Our notifier list
 */

240
static RAW_NOTIFIER_HEAD(netdev_chain);
L
Linus Torvalds 已提交
241 242 243 244 245

/*
 *	Device drivers call our routines to queue packets here. We empty the
 *	queue in the local softnet handler.
 */
246

247
DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
E
Eric Dumazet 已提交
248
EXPORT_PER_CPU_SYMBOL(softnet_data);
L
Linus Torvalds 已提交
249

250
#ifdef CONFIG_LOCKDEP
251
/*
252
 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
 * according to dev->type
 */
static const unsigned short netdev_lock_type[] =
	{ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
	 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
268 269 270
	 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
	 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
	 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
271

272
static const char *const netdev_lock_name[] =
273 274 275 276 277 278 279 280 281 282 283 284
	{"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
	 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
	 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
	 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
	 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
	 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
	 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
	 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
	 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
	 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
	 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
	 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
285 286 287
	 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
	 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
	 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
288 289

static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
290
static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
291 292 293 294 295 296 297 298 299 300 301 302

static inline unsigned short netdev_lock_pos(unsigned short dev_type)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
		if (netdev_lock_type[i] == dev_type)
			return i;
	/* the last key is used by default */
	return ARRAY_SIZE(netdev_lock_type) - 1;
}

303 304
static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
						 unsigned short dev_type)
305 306 307 308 309 310 311
{
	int i;

	i = netdev_lock_pos(dev_type);
	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
				   netdev_lock_name[i]);
}
312 313 314 315 316 317 318 319 320 321

static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
{
	int i;

	i = netdev_lock_pos(dev->type);
	lockdep_set_class_and_name(&dev->addr_list_lock,
				   &netdev_addr_lock_key[i],
				   netdev_lock_name[i]);
}
322
#else
323 324 325 326 327
static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
						 unsigned short dev_type)
{
}
static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
328 329 330
{
}
#endif
L
Linus Torvalds 已提交
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353

/*******************************************************************************

		Protocol management and registration routines

*******************************************************************************/

/*
 *	Add a protocol ID to the list. Now that the input handler is
 *	smarter we can dispense with all the messy stuff that used to be
 *	here.
 *
 *	BEWARE!!! Protocol handlers, mangling input packets,
 *	MUST BE last in hash buckets and checking protocol handlers
 *	MUST start from promiscuous ptype_all chain in net_bh.
 *	It is true now, do not change it.
 *	Explanation follows: if protocol handler, mangling packet, will
 *	be the first on list, it is not able to sense, that packet
 *	is cloned and should be copied-on-write, so that it will
 *	change it and subsequent readers will get broken packet.
 *							--ANK (980803)
 */

354 355 356 357 358 359 360 361
static inline struct list_head *ptype_head(const struct packet_type *pt)
{
	if (pt->type == htons(ETH_P_ALL))
		return &ptype_all;
	else
		return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
}

L
Linus Torvalds 已提交
362 363 364 365 366 367 368 369
/**
 *	dev_add_pack - add packet handler
 *	@pt: packet type declaration
 *
 *	Add a protocol handler to the networking stack. The passed &packet_type
 *	is linked into kernel lists and may not be freed until it has been
 *	removed from the kernel lists.
 *
370
 *	This call does not sleep therefore it can not
L
Linus Torvalds 已提交
371 372 373 374 375 376
 *	guarantee all CPU's that are in middle of receiving packets
 *	will see the new packet type (until the next received packet).
 */

void dev_add_pack(struct packet_type *pt)
{
377
	struct list_head *head = ptype_head(pt);
L
Linus Torvalds 已提交
378

379 380 381
	spin_lock(&ptype_lock);
	list_add_rcu(&pt->list, head);
	spin_unlock(&ptype_lock);
L
Linus Torvalds 已提交
382
}
E
Eric Dumazet 已提交
383
EXPORT_SYMBOL(dev_add_pack);
L
Linus Torvalds 已提交
384 385 386 387 388 389 390 391

/**
 *	__dev_remove_pack	 - remove packet handler
 *	@pt: packet type declaration
 *
 *	Remove a protocol handler that was previously added to the kernel
 *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
 *	from the kernel lists and can be freed or reused once this function
392
 *	returns.
L
Linus Torvalds 已提交
393 394 395 396 397 398 399
 *
 *      The packet type might still be in use by receivers
 *	and must not be freed until after all the CPU's have gone
 *	through a quiescent state.
 */
void __dev_remove_pack(struct packet_type *pt)
{
400
	struct list_head *head = ptype_head(pt);
L
Linus Torvalds 已提交
401 402
	struct packet_type *pt1;

403
	spin_lock(&ptype_lock);
L
Linus Torvalds 已提交
404 405 406 407 408 409 410 411

	list_for_each_entry(pt1, head, list) {
		if (pt == pt1) {
			list_del_rcu(&pt->list);
			goto out;
		}
	}

412
	pr_warn("dev_remove_pack: %p not found\n", pt);
L
Linus Torvalds 已提交
413
out:
414
	spin_unlock(&ptype_lock);
L
Linus Torvalds 已提交
415
}
E
Eric Dumazet 已提交
416 417
EXPORT_SYMBOL(__dev_remove_pack);

L
Linus Torvalds 已提交
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
/**
 *	dev_remove_pack	 - remove packet handler
 *	@pt: packet type declaration
 *
 *	Remove a protocol handler that was previously added to the kernel
 *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
 *	from the kernel lists and can be freed or reused once this function
 *	returns.
 *
 *	This call sleeps to guarantee that no CPU is looking at the packet
 *	type after return.
 */
void dev_remove_pack(struct packet_type *pt)
{
	__dev_remove_pack(pt);
433

L
Linus Torvalds 已提交
434 435
	synchronize_net();
}
E
Eric Dumazet 已提交
436
EXPORT_SYMBOL(dev_remove_pack);
L
Linus Torvalds 已提交
437

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478

/**
 *	dev_add_offload - register offload handlers
 *	@po: protocol offload declaration
 *
 *	Add protocol offload handlers to the networking stack. The passed
 *	&proto_offload is linked into kernel lists and may not be freed until
 *	it has been removed from the kernel lists.
 *
 *	This call does not sleep therefore it can not
 *	guarantee all CPU's that are in middle of receiving packets
 *	will see the new offload handlers (until the next received packet).
 */
void dev_add_offload(struct packet_offload *po)
{
	struct list_head *head = &offload_base;

	spin_lock(&offload_lock);
	list_add_rcu(&po->list, head);
	spin_unlock(&offload_lock);
}
EXPORT_SYMBOL(dev_add_offload);

/**
 *	__dev_remove_offload	 - remove offload handler
 *	@po: packet offload declaration
 *
 *	Remove a protocol offload handler that was previously added to the
 *	kernel offload handlers by dev_add_offload(). The passed &offload_type
 *	is removed from the kernel lists and can be freed or reused once this
 *	function returns.
 *
 *      The packet type might still be in use by receivers
 *	and must not be freed until after all the CPU's have gone
 *	through a quiescent state.
 */
void __dev_remove_offload(struct packet_offload *po)
{
	struct list_head *head = &offload_base;
	struct packet_offload *po1;

479
	spin_lock(&offload_lock);
480 481 482 483 484 485 486 487 488 489

	list_for_each_entry(po1, head, list) {
		if (po == po1) {
			list_del_rcu(&po->list);
			goto out;
		}
	}

	pr_warn("dev_remove_offload: %p not found\n", po);
out:
490
	spin_unlock(&offload_lock);
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
}
EXPORT_SYMBOL(__dev_remove_offload);

/**
 *	dev_remove_offload	 - remove packet offload handler
 *	@po: packet offload declaration
 *
 *	Remove a packet offload handler that was previously added to the kernel
 *	offload handlers by dev_add_offload(). The passed &offload_type is
 *	removed from the kernel lists and can be freed or reused once this
 *	function returns.
 *
 *	This call sleeps to guarantee that no CPU is looking at the packet
 *	type after return.
 */
void dev_remove_offload(struct packet_offload *po)
{
	__dev_remove_offload(po);

	synchronize_net();
}
EXPORT_SYMBOL(dev_remove_offload);

L
Linus Torvalds 已提交
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
/******************************************************************************

		      Device Boot-time Settings Routines

*******************************************************************************/

/* Boot time configuration table */
static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];

/**
 *	netdev_boot_setup_add	- add new setup entry
 *	@name: name of the device
 *	@map: configured settings for the device
 *
 *	Adds new setup entry to the dev_boot_setup list.  The function
 *	returns 0 on error and 1 on success.  This is a generic routine to
 *	all netdevices.
 */
static int netdev_boot_setup_add(char *name, struct ifmap *map)
{
	struct netdev_boot_setup *s;
	int i;

	s = dev_boot_setup;
	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
			memset(s[i].name, 0, sizeof(s[i].name));
541
			strlcpy(s[i].name, name, IFNAMSIZ);
L
Linus Torvalds 已提交
542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
			memcpy(&s[i].map, map, sizeof(s[i].map));
			break;
		}
	}

	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
}

/**
 *	netdev_boot_setup_check	- check boot time settings
 *	@dev: the netdevice
 *
 * 	Check boot time settings for the device.
 *	The found settings are set for the device to be used
 *	later in the device probing.
 *	Returns 0 if no settings found, 1 if they are.
 */
int netdev_boot_setup_check(struct net_device *dev)
{
	struct netdev_boot_setup *s = dev_boot_setup;
	int i;

	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
566
		    !strcmp(dev->name, s[i].name)) {
L
Linus Torvalds 已提交
567 568 569 570 571 572 573 574 575
			dev->irq 	= s[i].map.irq;
			dev->base_addr 	= s[i].map.base_addr;
			dev->mem_start 	= s[i].map.mem_start;
			dev->mem_end 	= s[i].map.mem_end;
			return 1;
		}
	}
	return 0;
}
E
Eric Dumazet 已提交
576
EXPORT_SYMBOL(netdev_boot_setup_check);
L
Linus Torvalds 已提交
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600


/**
 *	netdev_boot_base	- get address from boot time settings
 *	@prefix: prefix for network device
 *	@unit: id for network device
 *
 * 	Check boot time settings for the base address of device.
 *	The found settings are set for the device to be used
 *	later in the device probing.
 *	Returns 0 if no settings found.
 */
unsigned long netdev_boot_base(const char *prefix, int unit)
{
	const struct netdev_boot_setup *s = dev_boot_setup;
	char name[IFNAMSIZ];
	int i;

	sprintf(name, "%s%d", prefix, unit);

	/*
	 * If device already registered then return base of 1
	 * to indicate not to probe for this interface
	 */
601
	if (__dev_get_by_name(&init_net, name))
L
Linus Torvalds 已提交
602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
		return 1;

	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
		if (!strcmp(name, s[i].name))
			return s[i].map.base_addr;
	return 0;
}

/*
 * Saves at boot time configured settings for any netdevice.
 */
int __init netdev_boot_setup(char *str)
{
	int ints[5];
	struct ifmap map;

	str = get_options(str, ARRAY_SIZE(ints), ints);
	if (!str || !*str)
		return 0;

	/* Save settings */
	memset(&map, 0, sizeof(map));
	if (ints[0] > 0)
		map.irq = ints[1];
	if (ints[0] > 1)
		map.base_addr = ints[2];
	if (ints[0] > 2)
		map.mem_start = ints[3];
	if (ints[0] > 3)
		map.mem_end = ints[4];

	/* Add new entry to the list */
	return netdev_boot_setup_add(str, &map);
}

__setup("netdev=", netdev_boot_setup);

/*******************************************************************************

			    Device Interface Subroutines

*******************************************************************************/

/**
 *	__dev_get_by_name	- find a device by its name
647
 *	@net: the applicable net namespace
L
Linus Torvalds 已提交
648 649 650 651 652 653 654 655 656
 *	@name: name to find
 *
 *	Find an interface by name. Must be called under RTNL semaphore
 *	or @dev_base_lock. If the name is found a pointer to the device
 *	is returned. If the name is not found then %NULL is returned. The
 *	reference counters are not incremented so the caller must be
 *	careful with locks.
 */

657
struct net_device *__dev_get_by_name(struct net *net, const char *name)
L
Linus Torvalds 已提交
658
{
E
Eric Dumazet 已提交
659 660
	struct net_device *dev;
	struct hlist_head *head = dev_name_hash(net, name);
L
Linus Torvalds 已提交
661

662
	hlist_for_each_entry(dev, head, name_hlist)
L
Linus Torvalds 已提交
663 664
		if (!strncmp(dev->name, name, IFNAMSIZ))
			return dev;
E
Eric Dumazet 已提交
665

L
Linus Torvalds 已提交
666 667
	return NULL;
}
E
Eric Dumazet 已提交
668
EXPORT_SYMBOL(__dev_get_by_name);
L
Linus Torvalds 已提交
669

670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
/**
 *	dev_get_by_name_rcu	- find a device by its name
 *	@net: the applicable net namespace
 *	@name: name to find
 *
 *	Find an interface by name.
 *	If the name is found a pointer to the device is returned.
 * 	If the name is not found then %NULL is returned.
 *	The reference counters are not incremented so the caller must be
 *	careful with locks. The caller must hold RCU lock.
 */

struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
{
	struct net_device *dev;
	struct hlist_head *head = dev_name_hash(net, name);

687
	hlist_for_each_entry_rcu(dev, head, name_hlist)
688 689 690 691 692 693 694
		if (!strncmp(dev->name, name, IFNAMSIZ))
			return dev;

	return NULL;
}
EXPORT_SYMBOL(dev_get_by_name_rcu);

L
Linus Torvalds 已提交
695 696
/**
 *	dev_get_by_name		- find a device by its name
697
 *	@net: the applicable net namespace
L
Linus Torvalds 已提交
698 699 700 701 702 703 704 705 706
 *	@name: name to find
 *
 *	Find an interface by name. This can be called from any
 *	context and does its own locking. The returned handle has
 *	the usage count incremented and the caller must use dev_put() to
 *	release it when it is no longer needed. %NULL is returned if no
 *	matching device is found.
 */

707
struct net_device *dev_get_by_name(struct net *net, const char *name)
L
Linus Torvalds 已提交
708 709 710
{
	struct net_device *dev;

711 712
	rcu_read_lock();
	dev = dev_get_by_name_rcu(net, name);
L
Linus Torvalds 已提交
713 714
	if (dev)
		dev_hold(dev);
715
	rcu_read_unlock();
L
Linus Torvalds 已提交
716 717
	return dev;
}
E
Eric Dumazet 已提交
718
EXPORT_SYMBOL(dev_get_by_name);
L
Linus Torvalds 已提交
719 720 721

/**
 *	__dev_get_by_index - find a device by its ifindex
722
 *	@net: the applicable net namespace
L
Linus Torvalds 已提交
723 724 725 726 727 728 729 730 731
 *	@ifindex: index of device
 *
 *	Search for an interface by index. Returns %NULL if the device
 *	is not found or a pointer to the device. The device has not
 *	had its reference counter increased so the caller must be careful
 *	about locking. The caller must hold either the RTNL semaphore
 *	or @dev_base_lock.
 */

732
struct net_device *__dev_get_by_index(struct net *net, int ifindex)
L
Linus Torvalds 已提交
733
{
E
Eric Dumazet 已提交
734 735
	struct net_device *dev;
	struct hlist_head *head = dev_index_hash(net, ifindex);
L
Linus Torvalds 已提交
736

737
	hlist_for_each_entry(dev, head, index_hlist)
L
Linus Torvalds 已提交
738 739
		if (dev->ifindex == ifindex)
			return dev;
E
Eric Dumazet 已提交
740

L
Linus Torvalds 已提交
741 742
	return NULL;
}
E
Eric Dumazet 已提交
743
EXPORT_SYMBOL(__dev_get_by_index);
L
Linus Torvalds 已提交
744

745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
/**
 *	dev_get_by_index_rcu - find a device by its ifindex
 *	@net: the applicable net namespace
 *	@ifindex: index of device
 *
 *	Search for an interface by index. Returns %NULL if the device
 *	is not found or a pointer to the device. The device has not
 *	had its reference counter increased so the caller must be careful
 *	about locking. The caller must hold RCU lock.
 */

struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
{
	struct net_device *dev;
	struct hlist_head *head = dev_index_hash(net, ifindex);

761
	hlist_for_each_entry_rcu(dev, head, index_hlist)
762 763 764 765 766 767 768
		if (dev->ifindex == ifindex)
			return dev;

	return NULL;
}
EXPORT_SYMBOL(dev_get_by_index_rcu);

L
Linus Torvalds 已提交
769 770 771

/**
 *	dev_get_by_index - find a device by its ifindex
772
 *	@net: the applicable net namespace
L
Linus Torvalds 已提交
773 774 775 776 777 778 779 780
 *	@ifindex: index of device
 *
 *	Search for an interface by index. Returns NULL if the device
 *	is not found or a pointer to the device. The device returned has
 *	had a reference added and the pointer is safe until the user calls
 *	dev_put to indicate they have finished with it.
 */

781
struct net_device *dev_get_by_index(struct net *net, int ifindex)
L
Linus Torvalds 已提交
782 783 784
{
	struct net_device *dev;

785 786
	rcu_read_lock();
	dev = dev_get_by_index_rcu(net, ifindex);
L
Linus Torvalds 已提交
787 788
	if (dev)
		dev_hold(dev);
789
	rcu_read_unlock();
L
Linus Torvalds 已提交
790 791
	return dev;
}
E
Eric Dumazet 已提交
792
EXPORT_SYMBOL(dev_get_by_index);
L
Linus Torvalds 已提交
793 794

/**
795
 *	dev_getbyhwaddr_rcu - find a device by its hardware address
796
 *	@net: the applicable net namespace
L
Linus Torvalds 已提交
797 798 799 800
 *	@type: media type of device
 *	@ha: hardware address
 *
 *	Search for an interface by MAC address. Returns NULL if the device
E
Eric Dumazet 已提交
801 802
 *	is not found or a pointer to the device.
 *	The caller must hold RCU or RTNL.
803
 *	The returned device has not had its ref count increased
L
Linus Torvalds 已提交
804 805 806 807
 *	and the caller must therefore be careful about locking
 *
 */

808 809
struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
				       const char *ha)
L
Linus Torvalds 已提交
810 811 812
{
	struct net_device *dev;

813
	for_each_netdev_rcu(net, dev)
L
Linus Torvalds 已提交
814 815
		if (dev->type == type &&
		    !memcmp(dev->dev_addr, ha, dev->addr_len))
816 817 818
			return dev;

	return NULL;
L
Linus Torvalds 已提交
819
}
820
EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
821

822
struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
L
Linus Torvalds 已提交
823 824 825
{
	struct net_device *dev;

826
	ASSERT_RTNL();
827
	for_each_netdev(net, dev)
828
		if (dev->type == type)
829 830 831
			return dev;

	return NULL;
832 833 834
}
EXPORT_SYMBOL(__dev_getfirstbyhwtype);

835
struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
836
{
837
	struct net_device *dev, *ret = NULL;
838

839 840 841 842 843 844 845 846 847
	rcu_read_lock();
	for_each_netdev_rcu(net, dev)
		if (dev->type == type) {
			dev_hold(dev);
			ret = dev;
			break;
		}
	rcu_read_unlock();
	return ret;
L
Linus Torvalds 已提交
848 849 850 851
}
EXPORT_SYMBOL(dev_getfirstbyhwtype);

/**
E
Eric Dumazet 已提交
852
 *	dev_get_by_flags_rcu - find any device with given flags
853
 *	@net: the applicable net namespace
L
Linus Torvalds 已提交
854 855 856 857
 *	@if_flags: IFF_* values
 *	@mask: bitmask of bits in if_flags to check
 *
 *	Search for any interface with the given flags. Returns NULL if a device
E
Eric Dumazet 已提交
858 859
 *	is not found or a pointer to the device. Must be called inside
 *	rcu_read_lock(), and result refcount is unchanged.
L
Linus Torvalds 已提交
860 861
 */

E
Eric Dumazet 已提交
862
struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
E
Eric Dumazet 已提交
863
				    unsigned short mask)
L
Linus Torvalds 已提交
864
{
865
	struct net_device *dev, *ret;
L
Linus Torvalds 已提交
866

867
	ret = NULL;
868
	for_each_netdev_rcu(net, dev) {
L
Linus Torvalds 已提交
869
		if (((dev->flags ^ if_flags) & mask) == 0) {
870
			ret = dev;
L
Linus Torvalds 已提交
871 872 873
			break;
		}
	}
874
	return ret;
L
Linus Torvalds 已提交
875
}
E
Eric Dumazet 已提交
876
EXPORT_SYMBOL(dev_get_by_flags_rcu);
L
Linus Torvalds 已提交
877 878 879 880 881 882

/**
 *	dev_valid_name - check if name is okay for network device
 *	@name: name string
 *
 *	Network device names need to be valid file names to
883 884
 *	to allow sysfs to work.  We also disallow any kind of
 *	whitespace.
L
Linus Torvalds 已提交
885
 */
886
bool dev_valid_name(const char *name)
L
Linus Torvalds 已提交
887
{
888
	if (*name == '\0')
889
		return false;
890
	if (strlen(name) >= IFNAMSIZ)
891
		return false;
892
	if (!strcmp(name, ".") || !strcmp(name, ".."))
893
		return false;
894 895 896

	while (*name) {
		if (*name == '/' || isspace(*name))
897
			return false;
898 899
		name++;
	}
900
	return true;
L
Linus Torvalds 已提交
901
}
E
Eric Dumazet 已提交
902
EXPORT_SYMBOL(dev_valid_name);
L
Linus Torvalds 已提交
903 904

/**
905 906
 *	__dev_alloc_name - allocate a name for a device
 *	@net: network namespace to allocate the device name in
L
Linus Torvalds 已提交
907
 *	@name: name format string
908
 *	@buf:  scratch buffer and result name string
L
Linus Torvalds 已提交
909 910
 *
 *	Passed a format string - eg "lt%d" it will try and find a suitable
S
Stephen Hemminger 已提交
911 912 913 914 915 916
 *	id. It scans list of devices to build up a free map, then chooses
 *	the first empty slot. The caller must hold the dev_base or rtnl lock
 *	while allocating the name and adding the device in order to avoid
 *	duplicates.
 *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
 *	Returns the number of the unit assigned or a negative errno code.
L
Linus Torvalds 已提交
917 918
 */

919
static int __dev_alloc_name(struct net *net, const char *name, char *buf)
L
Linus Torvalds 已提交
920 921 922 923
{
	int i = 0;
	const char *p;
	const int max_netdevices = 8*PAGE_SIZE;
S
Stephen Hemminger 已提交
924
	unsigned long *inuse;
L
Linus Torvalds 已提交
925 926 927 928 929 930 931 932 933 934 935 936 937
	struct net_device *d;

	p = strnchr(name, IFNAMSIZ-1, '%');
	if (p) {
		/*
		 * Verify the string as this thing may have come from
		 * the user.  There must be either one "%d" and no other "%"
		 * characters.
		 */
		if (p[1] != 'd' || strchr(p + 2, '%'))
			return -EINVAL;

		/* Use one page as a bit array of possible slots */
S
Stephen Hemminger 已提交
938
		inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
L
Linus Torvalds 已提交
939 940 941
		if (!inuse)
			return -ENOMEM;

942
		for_each_netdev(net, d) {
L
Linus Torvalds 已提交
943 944 945 946 947 948
			if (!sscanf(d->name, name, &i))
				continue;
			if (i < 0 || i >= max_netdevices)
				continue;

			/*  avoid cases where sscanf is not exact inverse of printf */
949
			snprintf(buf, IFNAMSIZ, name, i);
L
Linus Torvalds 已提交
950 951 952 953 954 955 956 957
			if (!strncmp(buf, d->name, IFNAMSIZ))
				set_bit(i, inuse);
		}

		i = find_first_zero_bit(inuse, max_netdevices);
		free_page((unsigned long) inuse);
	}

958 959
	if (buf != name)
		snprintf(buf, IFNAMSIZ, name, i);
960
	if (!__dev_get_by_name(net, buf))
L
Linus Torvalds 已提交
961 962 963 964 965 966 967 968 969
		return i;

	/* It is possible to run out of possible slots
	 * when the name is long and there isn't enough space left
	 * for the digits, or if all bits are used.
	 */
	return -ENFILE;
}

970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989
/**
 *	dev_alloc_name - allocate a name for a device
 *	@dev: device
 *	@name: name format string
 *
 *	Passed a format string - eg "lt%d" it will try and find a suitable
 *	id. It scans list of devices to build up a free map, then chooses
 *	the first empty slot. The caller must hold the dev_base or rtnl lock
 *	while allocating the name and adding the device in order to avoid
 *	duplicates.
 *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
 *	Returns the number of the unit assigned or a negative errno code.
 */

int dev_alloc_name(struct net_device *dev, const char *name)
{
	char buf[IFNAMSIZ];
	struct net *net;
	int ret;

990 991
	BUG_ON(!dev_net(dev));
	net = dev_net(dev);
992 993 994 995 996
	ret = __dev_alloc_name(net, name, buf);
	if (ret >= 0)
		strlcpy(dev->name, buf, IFNAMSIZ);
	return ret;
}
E
Eric Dumazet 已提交
997
EXPORT_SYMBOL(dev_alloc_name);
998

999 1000 1001
static int dev_alloc_name_ns(struct net *net,
			     struct net_device *dev,
			     const char *name)
1002
{
1003 1004
	char buf[IFNAMSIZ];
	int ret;
1005

1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
	ret = __dev_alloc_name(net, name, buf);
	if (ret >= 0)
		strlcpy(dev->name, buf, IFNAMSIZ);
	return ret;
}

static int dev_get_valid_name(struct net *net,
			      struct net_device *dev,
			      const char *name)
{
	BUG_ON(!net);
1017

1018 1019 1020
	if (!dev_valid_name(name))
		return -EINVAL;

1021
	if (strchr(name, '%'))
1022
		return dev_alloc_name_ns(net, dev, name);
1023 1024
	else if (__dev_get_by_name(net, name))
		return -EEXIST;
1025 1026
	else if (dev->name != name)
		strlcpy(dev->name, name, IFNAMSIZ);
1027 1028 1029

	return 0;
}
L
Linus Torvalds 已提交
1030 1031 1032 1033 1034 1035 1036 1037 1038

/**
 *	dev_change_name - change name of a device
 *	@dev: device
 *	@newname: name (or format string) must be at least IFNAMSIZ
 *
 *	Change name of a device, can pass format strings "eth%d".
 *	for wildcarding.
 */
1039
int dev_change_name(struct net_device *dev, const char *newname)
L
Linus Torvalds 已提交
1040
{
1041
	char oldname[IFNAMSIZ];
L
Linus Torvalds 已提交
1042
	int err = 0;
1043
	int ret;
1044
	struct net *net;
L
Linus Torvalds 已提交
1045 1046

	ASSERT_RTNL();
1047
	BUG_ON(!dev_net(dev));
L
Linus Torvalds 已提交
1048

1049
	net = dev_net(dev);
L
Linus Torvalds 已提交
1050 1051 1052
	if (dev->flags & IFF_UP)
		return -EBUSY;

1053
	write_seqcount_begin(&devnet_rename_seq);
1054 1055

	if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1056
		write_seqcount_end(&devnet_rename_seq);
1057
		return 0;
1058
	}
1059

1060 1061
	memcpy(oldname, dev->name, IFNAMSIZ);

1062
	err = dev_get_valid_name(net, dev, newname);
1063
	if (err < 0) {
1064
		write_seqcount_end(&devnet_rename_seq);
1065
		return err;
1066
	}
L
Linus Torvalds 已提交
1067

1068
rollback:
1069 1070 1071
	ret = device_rename(&dev->dev, dev->name);
	if (ret) {
		memcpy(dev->name, oldname, IFNAMSIZ);
1072
		write_seqcount_end(&devnet_rename_seq);
1073
		return ret;
1074
	}
1075

1076
	write_seqcount_end(&devnet_rename_seq);
1077

1078
	write_lock_bh(&dev_base_lock);
1079
	hlist_del_rcu(&dev->name_hlist);
1080 1081 1082 1083 1084 1085
	write_unlock_bh(&dev_base_lock);

	synchronize_rcu();

	write_lock_bh(&dev_base_lock);
	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1086 1087
	write_unlock_bh(&dev_base_lock);

1088
	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1089 1090 1091
	ret = notifier_to_errno(ret);

	if (ret) {
1092 1093
		/* err >= 0 after dev_alloc_name() or stores the first errno */
		if (err >= 0) {
1094
			err = ret;
1095
			write_seqcount_begin(&devnet_rename_seq);
1096 1097
			memcpy(dev->name, oldname, IFNAMSIZ);
			goto rollback;
1098
		} else {
1099
			pr_err("%s: name change rollback failed: %d\n",
1100
			       dev->name, ret);
1101 1102
		}
	}
L
Linus Torvalds 已提交
1103 1104 1105 1106

	return err;
}

1107 1108 1109 1110
/**
 *	dev_set_alias - change ifalias of a device
 *	@dev: device
 *	@alias: name up to IFALIASZ
1111
 *	@len: limit of bytes to copy from info
1112 1113 1114 1115 1116
 *
 *	Set ifalias for a device,
 */
int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
{
1117 1118
	char *new_ifalias;

1119 1120 1121 1122 1123
	ASSERT_RTNL();

	if (len >= IFALIASZ)
		return -EINVAL;

1124
	if (!len) {
1125 1126
		kfree(dev->ifalias);
		dev->ifalias = NULL;
1127 1128 1129
		return 0;
	}

1130 1131
	new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
	if (!new_ifalias)
1132
		return -ENOMEM;
1133
	dev->ifalias = new_ifalias;
1134 1135 1136 1137 1138 1139

	strlcpy(dev->ifalias, alias, len+1);
	return len;
}


1140
/**
S
Stephen Hemminger 已提交
1141
 *	netdev_features_change - device changes features
1142 1143 1144 1145 1146 1147
 *	@dev: device to cause notification
 *
 *	Called to indicate a device has changed features.
 */
void netdev_features_change(struct net_device *dev)
{
1148
	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1149 1150 1151
}
EXPORT_SYMBOL(netdev_features_change);

L
Linus Torvalds 已提交
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
/**
 *	netdev_state_change - device changes state
 *	@dev: device to cause notification
 *
 *	Called to indicate a device has changed state. This function calls
 *	the notifier chains for netdev_chain and sends a NEWLINK message
 *	to the routing socket.
 */
void netdev_state_change(struct net_device *dev)
{
	if (dev->flags & IFF_UP) {
1163
		call_netdevice_notifiers(NETDEV_CHANGE, dev);
L
Linus Torvalds 已提交
1164 1165 1166
		rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
	}
}
E
Eric Dumazet 已提交
1167
EXPORT_SYMBOL(netdev_state_change);
L
Linus Torvalds 已提交
1168

1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
/**
 * 	netdev_notify_peers - notify network peers about existence of @dev
 * 	@dev: network device
 *
 * Generate traffic such that interested network peers are aware of
 * @dev, such as by generating a gratuitous ARP. This may be used when
 * a device wants to inform the rest of the network about some sort of
 * reconfiguration such as a failover event or virtual machine
 * migration.
 */
void netdev_notify_peers(struct net_device *dev)
1180
{
1181 1182 1183
	rtnl_lock();
	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
	rtnl_unlock();
1184
}
1185
EXPORT_SYMBOL(netdev_notify_peers);
1186

1187
static int __dev_open(struct net_device *dev)
L
Linus Torvalds 已提交
1188
{
1189
	const struct net_device_ops *ops = dev->netdev_ops;
1190
	int ret;
L
Linus Torvalds 已提交
1191

1192 1193
	ASSERT_RTNL();

L
Linus Torvalds 已提交
1194 1195 1196
	if (!netif_device_present(dev))
		return -ENODEV;

1197 1198 1199 1200 1201 1202 1203 1204
	/* Block netpoll from trying to do any rx path servicing.
	 * If we don't do this there is a chance ndo_poll_controller
	 * or ndo_poll may be running while we open the device
	 */
	ret = netpoll_rx_disable(dev);
	if (ret)
		return ret;

1205 1206 1207 1208 1209
	ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
	ret = notifier_to_errno(ret);
	if (ret)
		return ret;

L
Linus Torvalds 已提交
1210
	set_bit(__LINK_STATE_START, &dev->state);
1211

1212 1213
	if (ops->ndo_validate_addr)
		ret = ops->ndo_validate_addr(dev);
1214

1215 1216
	if (!ret && ops->ndo_open)
		ret = ops->ndo_open(dev);
L
Linus Torvalds 已提交
1217

1218 1219
	netpoll_rx_enable(dev);

1220 1221 1222
	if (ret)
		clear_bit(__LINK_STATE_START, &dev->state);
	else {
L
Linus Torvalds 已提交
1223
		dev->flags |= IFF_UP;
1224
		net_dmaengine_get();
1225
		dev_set_rx_mode(dev);
L
Linus Torvalds 已提交
1226
		dev_activate(dev);
1227
		add_device_randomness(dev->dev_addr, dev->addr_len);
L
Linus Torvalds 已提交
1228
	}
1229

L
Linus Torvalds 已提交
1230 1231 1232 1233
	return ret;
}

/**
1234 1235
 *	dev_open	- prepare an interface for use.
 *	@dev:	device to open
L
Linus Torvalds 已提交
1236
 *
1237 1238 1239 1240 1241 1242 1243
 *	Takes a device from down to up state. The device's private open
 *	function is invoked and then the multicast lists are loaded. Finally
 *	the device is moved into the up state and a %NETDEV_UP message is
 *	sent to the netdev notifier chain.
 *
 *	Calling this function on an active interface is a nop. On a failure
 *	a negative errno code is returned.
L
Linus Torvalds 已提交
1244
 */
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
int dev_open(struct net_device *dev)
{
	int ret;

	if (dev->flags & IFF_UP)
		return 0;

	ret = __dev_open(dev);
	if (ret < 0)
		return ret;

	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
	call_netdevice_notifiers(NETDEV_UP, dev);

	return ret;
}
EXPORT_SYMBOL(dev_open);

1263
static int __dev_close_many(struct list_head *head)
L
Linus Torvalds 已提交
1264
{
1265
	struct net_device *dev;
1266

1267
	ASSERT_RTNL();
1268 1269
	might_sleep();

1270 1271
	list_for_each_entry(dev, head, unreg_list) {
		call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
L
Linus Torvalds 已提交
1272

1273
		clear_bit(__LINK_STATE_START, &dev->state);
L
Linus Torvalds 已提交
1274

1275 1276 1277 1278 1279 1280 1281 1282
		/* Synchronize to scheduled poll. We cannot touch poll list, it
		 * can be even on different cpu. So just clear netif_running().
		 *
		 * dev->stop() will invoke napi_disable() on all of it's
		 * napi_struct instances on this device.
		 */
		smp_mb__after_clear_bit(); /* Commit netif_running(). */
	}
L
Linus Torvalds 已提交
1283

1284
	dev_deactivate_many(head);
1285

1286 1287
	list_for_each_entry(dev, head, unreg_list) {
		const struct net_device_ops *ops = dev->netdev_ops;
L
Linus Torvalds 已提交
1288

1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
		/*
		 *	Call the device specific close. This cannot fail.
		 *	Only if device is UP
		 *
		 *	We allow it to be called even after a DETACH hot-plug
		 *	event.
		 */
		if (ops->ndo_stop)
			ops->ndo_stop(dev);

		dev->flags &= ~IFF_UP;
		net_dmaengine_put();
	}

	return 0;
}

static int __dev_close(struct net_device *dev)
{
1308
	int retval;
1309 1310
	LIST_HEAD(single);

1311 1312 1313 1314 1315
	/* Temporarily disable netpoll until the interface is down */
	retval = netpoll_rx_disable(dev);
	if (retval)
		return retval;

1316
	list_add(&dev->unreg_list, &single);
1317 1318
	retval = __dev_close_many(&single);
	list_del(&single);
1319 1320

	netpoll_rx_enable(dev);
1321
	return retval;
1322 1323
}

E
Eric Dumazet 已提交
1324
static int dev_close_many(struct list_head *head)
1325 1326 1327
{
	struct net_device *dev, *tmp;
	LIST_HEAD(tmp_list);
L
Linus Torvalds 已提交
1328

1329 1330 1331 1332 1333
	list_for_each_entry_safe(dev, tmp, head, unreg_list)
		if (!(dev->flags & IFF_UP))
			list_move(&dev->unreg_list, &tmp_list);

	__dev_close_many(head);
L
Linus Torvalds 已提交
1334

1335 1336 1337 1338
	list_for_each_entry(dev, head, unreg_list) {
		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
		call_netdevice_notifiers(NETDEV_DOWN, dev);
	}
1339

1340 1341
	/* rollback_registered_many needs the complete original list */
	list_splice(&tmp_list, head);
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
	return 0;
}

/**
 *	dev_close - shutdown an interface.
 *	@dev: device to shutdown
 *
 *	This function moves an active device into down state. A
 *	%NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
 *	is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
 *	chain.
 */
int dev_close(struct net_device *dev)
{
1356
	int ret = 0;
1357 1358
	if (dev->flags & IFF_UP) {
		LIST_HEAD(single);
L
Linus Torvalds 已提交
1359

1360 1361 1362 1363 1364
		/* Block netpoll rx while the interface is going down */
		ret = netpoll_rx_disable(dev);
		if (ret)
			return ret;

1365 1366 1367
		list_add(&dev->unreg_list, &single);
		dev_close_many(&single);
		list_del(&single);
1368 1369

		netpoll_rx_enable(dev);
1370
	}
1371
	return ret;
L
Linus Torvalds 已提交
1372
}
E
Eric Dumazet 已提交
1373
EXPORT_SYMBOL(dev_close);
L
Linus Torvalds 已提交
1374 1375


1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
/**
 *	dev_disable_lro - disable Large Receive Offload on a device
 *	@dev: device
 *
 *	Disable Large Receive Offload (LRO) on a net device.  Must be
 *	called under RTNL.  This is needed if received packets may be
 *	forwarded to another interface.
 */
void dev_disable_lro(struct net_device *dev)
{
1386 1387 1388 1389 1390 1391 1392
	/*
	 * If we're trying to disable lro on a vlan device
	 * use the underlying physical device instead
	 */
	if (is_vlan_dev(dev))
		dev = vlan_dev_real_dev(dev);

M
Michał Mirosław 已提交
1393 1394
	dev->wanted_features &= ~NETIF_F_LRO;
	netdev_update_features(dev);
1395

1396 1397
	if (unlikely(dev->features & NETIF_F_LRO))
		netdev_WARN(dev, "failed to disable LRO!\n");
1398 1399 1400 1401
}
EXPORT_SYMBOL(dev_disable_lro);


1402 1403
static int dev_boot_phase = 1;

L
Linus Torvalds 已提交
1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
/**
 *	register_netdevice_notifier - register a network notifier block
 *	@nb: notifier
 *
 *	Register a notifier to be called when network device events occur.
 *	The notifier passed is linked into the kernel structures and must
 *	not be reused until it has been unregistered. A negative errno code
 *	is returned on a failure.
 *
 * 	When registered all registration and up events are replayed
1414
 *	to the new notifier to allow device to have a race free
L
Linus Torvalds 已提交
1415 1416 1417 1418 1419 1420
 *	view of the network device list.
 */

int register_netdevice_notifier(struct notifier_block *nb)
{
	struct net_device *dev;
1421
	struct net_device *last;
1422
	struct net *net;
L
Linus Torvalds 已提交
1423 1424 1425
	int err;

	rtnl_lock();
1426
	err = raw_notifier_chain_register(&netdev_chain, nb);
1427 1428
	if (err)
		goto unlock;
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
	if (dev_boot_phase)
		goto unlock;
	for_each_net(net) {
		for_each_netdev(net, dev) {
			err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
			err = notifier_to_errno(err);
			if (err)
				goto rollback;

			if (!(dev->flags & IFF_UP))
				continue;
L
Linus Torvalds 已提交
1440

1441 1442
			nb->notifier_call(nb, NETDEV_UP, dev);
		}
L
Linus Torvalds 已提交
1443
	}
1444 1445

unlock:
L
Linus Torvalds 已提交
1446 1447
	rtnl_unlock();
	return err;
1448 1449 1450

rollback:
	last = dev;
1451 1452 1453
	for_each_net(net) {
		for_each_netdev(net, dev) {
			if (dev == last)
1454
				goto outroll;
1455

1456 1457 1458 1459 1460
			if (dev->flags & IFF_UP) {
				nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
				nb->notifier_call(nb, NETDEV_DOWN, dev);
			}
			nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1461 1462
		}
	}
1463

1464
outroll:
1465
	raw_notifier_chain_unregister(&netdev_chain, nb);
1466
	goto unlock;
L
Linus Torvalds 已提交
1467
}
E
Eric Dumazet 已提交
1468
EXPORT_SYMBOL(register_netdevice_notifier);
L
Linus Torvalds 已提交
1469 1470 1471 1472 1473 1474 1475 1476 1477

/**
 *	unregister_netdevice_notifier - unregister a network notifier block
 *	@nb: notifier
 *
 *	Unregister a notifier previously registered by
 *	register_netdevice_notifier(). The notifier is unlinked into the
 *	kernel structures and may then be reused. A negative errno code
 *	is returned on a failure.
1478 1479 1480 1481
 *
 * 	After unregistering unregister and down device events are synthesized
 *	for all devices on the device list to the removed notifier to remove
 *	the need for special case cleanup code.
L
Linus Torvalds 已提交
1482 1483 1484 1485
 */

int unregister_netdevice_notifier(struct notifier_block *nb)
{
1486 1487
	struct net_device *dev;
	struct net *net;
1488 1489 1490
	int err;

	rtnl_lock();
1491
	err = raw_notifier_chain_unregister(&netdev_chain, nb);
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504
	if (err)
		goto unlock;

	for_each_net(net) {
		for_each_netdev(net, dev) {
			if (dev->flags & IFF_UP) {
				nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
				nb->notifier_call(nb, NETDEV_DOWN, dev);
			}
			nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
		}
	}
unlock:
1505 1506
	rtnl_unlock();
	return err;
L
Linus Torvalds 已提交
1507
}
E
Eric Dumazet 已提交
1508
EXPORT_SYMBOL(unregister_netdevice_notifier);
L
Linus Torvalds 已提交
1509 1510 1511 1512

/**
 *	call_netdevice_notifiers - call all network notifier blocks
 *      @val: value passed unmodified to notifier function
1513
 *      @dev: net_device pointer passed unmodified to notifier function
L
Linus Torvalds 已提交
1514 1515
 *
 *	Call all network notifier blocks.  Parameters and return value
1516
 *	are as for raw_notifier_call_chain().
L
Linus Torvalds 已提交
1517 1518
 */

1519
int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
L
Linus Torvalds 已提交
1520
{
1521
	ASSERT_RTNL();
1522
	return raw_notifier_call_chain(&netdev_chain, val, dev);
L
Linus Torvalds 已提交
1523
}
1524
EXPORT_SYMBOL(call_netdevice_notifiers);
L
Linus Torvalds 已提交
1525

1526
static struct static_key netstamp_needed __read_mostly;
1527
#ifdef HAVE_JUMP_LABEL
1528
/* We are not allowed to call static_key_slow_dec() from irq context
1529
 * If net_disable_timestamp() is called from irq context, defer the
1530
 * static_key_slow_dec() calls.
1531 1532 1533
 */
static atomic_t netstamp_needed_deferred;
#endif
L
Linus Torvalds 已提交
1534 1535 1536

void net_enable_timestamp(void)
{
1537 1538 1539 1540 1541
#ifdef HAVE_JUMP_LABEL
	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);

	if (deferred) {
		while (--deferred)
1542
			static_key_slow_dec(&netstamp_needed);
1543 1544 1545
		return;
	}
#endif
1546
	static_key_slow_inc(&netstamp_needed);
L
Linus Torvalds 已提交
1547
}
E
Eric Dumazet 已提交
1548
EXPORT_SYMBOL(net_enable_timestamp);
L
Linus Torvalds 已提交
1549 1550 1551

void net_disable_timestamp(void)
{
1552 1553 1554 1555 1556 1557
#ifdef HAVE_JUMP_LABEL
	if (in_interrupt()) {
		atomic_inc(&netstamp_needed_deferred);
		return;
	}
#endif
1558
	static_key_slow_dec(&netstamp_needed);
L
Linus Torvalds 已提交
1559
}
E
Eric Dumazet 已提交
1560
EXPORT_SYMBOL(net_disable_timestamp);
L
Linus Torvalds 已提交
1561

E
Eric Dumazet 已提交
1562
static inline void net_timestamp_set(struct sk_buff *skb)
L
Linus Torvalds 已提交
1563
{
1564
	skb->tstamp.tv64 = 0;
1565
	if (static_key_false(&netstamp_needed))
1566
		__net_timestamp(skb);
L
Linus Torvalds 已提交
1567 1568
}

1569
#define net_timestamp_check(COND, SKB)			\
1570
	if (static_key_false(&netstamp_needed)) {		\
1571 1572 1573
		if ((COND) && !(SKB)->tstamp.tv64)	\
			__net_timestamp(SKB);		\
	}						\
E
Eric Dumazet 已提交
1574

1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
static inline bool is_skb_forwardable(struct net_device *dev,
				      struct sk_buff *skb)
{
	unsigned int len;

	if (!(dev->flags & IFF_UP))
		return false;

	len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
	if (skb->len <= len)
		return true;

	/* if TSO is enabled, we don't care about the length as the packet
	 * could be forwarded without being segmented before
	 */
	if (skb_is_gso(skb))
		return true;

	return false;
}

1596 1597 1598 1599 1600 1601 1602 1603
/**
 * dev_forward_skb - loopback an skb to another netif
 *
 * @dev: destination network device
 * @skb: buffer to forward
 *
 * return values:
 *	NET_RX_SUCCESS	(no congestion)
1604
 *	NET_RX_DROP     (packet was dropped, but freed)
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
 *
 * dev_forward_skb can be used for injecting an skb from the
 * start_xmit function of one device into the receive queue
 * of another device.
 *
 * The receiving device may be in another namespace, so
 * we have to clear all information in the skb that could
 * impact namespace isolation.
 */
int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
{
1616 1617 1618 1619 1620 1621 1622 1623
	if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
		if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
			atomic_long_inc(&dev->rx_dropped);
			kfree_skb(skb);
			return NET_RX_DROP;
		}
	}

1624 1625
	skb_orphan(skb);

1626
	if (unlikely(!is_skb_forwardable(dev, skb))) {
1627
		atomic_long_inc(&dev->rx_dropped);
1628
		kfree_skb(skb);
1629
		return NET_RX_DROP;
1630
	}
1631
	skb->skb_iif = 0;
1632
	skb_dst_drop(skb);
1633 1634 1635
	skb->tstamp.tv64 = 0;
	skb->pkt_type = PACKET_HOST;
	skb->protocol = eth_type_trans(skb, dev);
1636 1637 1638
	skb->mark = 0;
	secpath_reset(skb);
	nf_reset(skb);
1639
	nf_reset_trace(skb);
1640 1641 1642 1643
	return netif_rx(skb);
}
EXPORT_SYMBOL_GPL(dev_forward_skb);

1644 1645 1646 1647
static inline int deliver_skb(struct sk_buff *skb,
			      struct packet_type *pt_prev,
			      struct net_device *orig_dev)
{
1648 1649
	if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
		return -ENOMEM;
1650 1651 1652 1653
	atomic_inc(&skb->users);
	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
}

1654 1655
static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
{
1656
	if (!ptype->af_packet_priv || !skb->sk)
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
		return false;

	if (ptype->id_match)
		return ptype->id_match(ptype, skb->sk);
	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
		return true;

	return false;
}

L
Linus Torvalds 已提交
1667 1668 1669 1670 1671
/*
 *	Support routine. Sends outgoing frames to any network
 *	taps currently in use.
 */

1672
static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
L
Linus Torvalds 已提交
1673 1674
{
	struct packet_type *ptype;
1675 1676
	struct sk_buff *skb2 = NULL;
	struct packet_type *pt_prev = NULL;
1677

L
Linus Torvalds 已提交
1678 1679 1680 1681 1682 1683
	rcu_read_lock();
	list_for_each_entry_rcu(ptype, &ptype_all, list) {
		/* Never send packets back to the socket
		 * they originated from - MvS (miquels@drinkel.ow.org)
		 */
		if ((ptype->dev == dev || !ptype->dev) &&
1684
		    (!skb_loop_sk(ptype, skb))) {
1685 1686 1687 1688 1689 1690 1691
			if (pt_prev) {
				deliver_skb(skb2, pt_prev, skb->dev);
				pt_prev = ptype;
				continue;
			}

			skb2 = skb_clone(skb, GFP_ATOMIC);
L
Linus Torvalds 已提交
1692 1693 1694
			if (!skb2)
				break;

1695 1696
			net_timestamp_set(skb2);

L
Linus Torvalds 已提交
1697 1698 1699 1700
			/* skb->nh should be correctly
			   set by sender, so that the second statement is
			   just protection against buggy protocols.
			 */
1701
			skb_reset_mac_header(skb2);
L
Linus Torvalds 已提交
1702

1703
			if (skb_network_header(skb2) < skb2->data ||
1704
			    skb2->network_header > skb2->tail) {
1705 1706 1707
				net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
						     ntohs(skb2->protocol),
						     dev->name);
1708
				skb_reset_network_header(skb2);
L
Linus Torvalds 已提交
1709 1710
			}

1711
			skb2->transport_header = skb2->network_header;
L
Linus Torvalds 已提交
1712
			skb2->pkt_type = PACKET_OUTGOING;
1713
			pt_prev = ptype;
L
Linus Torvalds 已提交
1714 1715
		}
	}
1716 1717
	if (pt_prev)
		pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
L
Linus Torvalds 已提交
1718 1719 1720
	rcu_read_unlock();
}

1721 1722
/**
 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
 * @dev: Network device
 * @txq: number of queues available
 *
 * If real_num_tx_queues is changed the tc mappings may no longer be
 * valid. To resolve this verify the tc mapping remains valid and if
 * not NULL the mapping. With no priorities mapping to this
 * offset/count pair it will no longer be used. In the worst case TC0
 * is invalid nothing can be done so disable priority mappings. If is
 * expected that drivers will fix this mapping if they can before
 * calling netif_set_real_num_tx_queues.
 */
E
Eric Dumazet 已提交
1734
static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1735 1736 1737 1738 1739 1740
{
	int i;
	struct netdev_tc_txq *tc = &dev->tc_to_txq[0];

	/* If TC0 is invalidated disable TC mapping */
	if (tc->offset + tc->count > txq) {
1741
		pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751
		dev->num_tc = 0;
		return;
	}

	/* Invalidated prio to tc mappings set to TC0 */
	for (i = 1; i < TC_BITMASK + 1; i++) {
		int q = netdev_get_prio_tc_map(dev, i);

		tc = &dev->tc_to_txq[q];
		if (tc->offset + tc->count > txq) {
1752 1753
			pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
				i, q);
1754 1755 1756 1757 1758
			netdev_set_prio_tc_map(dev, i, 0);
		}
	}
}

1759 1760 1761 1762 1763
#ifdef CONFIG_XPS
static DEFINE_MUTEX(xps_map_mutex);
#define xmap_dereference(P)		\
	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))

1764 1765
static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
					int cpu, u16 index)
1766
{
1767 1768
	struct xps_map *map = NULL;
	int pos;
1769

1770 1771
	if (dev_maps)
		map = xmap_dereference(dev_maps->cpu_map[cpu]);
1772

1773 1774
	for (pos = 0; map && pos < map->len; pos++) {
		if (map->queues[pos] == index) {
1775 1776 1777
			if (map->len > 1) {
				map->queues[pos] = map->queues[--map->len];
			} else {
1778
				RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1779 1780 1781
				kfree_rcu(map, rcu);
				map = NULL;
			}
1782
			break;
1783 1784 1785
		}
	}

1786 1787 1788
	return map;
}

1789
static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1790 1791
{
	struct xps_dev_maps *dev_maps;
1792
	int cpu, i;
1793 1794 1795 1796 1797 1798 1799 1800 1801
	bool active = false;

	mutex_lock(&xps_map_mutex);
	dev_maps = xmap_dereference(dev->xps_maps);

	if (!dev_maps)
		goto out_no_maps;

	for_each_possible_cpu(cpu) {
1802 1803 1804 1805 1806
		for (i = index; i < dev->num_tx_queues; i++) {
			if (!remove_xps_queue(dev_maps, cpu, i))
				break;
		}
		if (i == dev->num_tx_queues)
1807 1808 1809 1810
			active = true;
	}

	if (!active) {
1811 1812 1813 1814
		RCU_INIT_POINTER(dev->xps_maps, NULL);
		kfree_rcu(dev_maps, rcu);
	}

1815 1816 1817 1818
	for (i = index; i < dev->num_tx_queues; i++)
		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
					     NUMA_NO_NODE);

1819 1820 1821 1822
out_no_maps:
	mutex_unlock(&xps_map_mutex);
}

1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857
static struct xps_map *expand_xps_map(struct xps_map *map,
				      int cpu, u16 index)
{
	struct xps_map *new_map;
	int alloc_len = XPS_MIN_MAP_ALLOC;
	int i, pos;

	for (pos = 0; map && pos < map->len; pos++) {
		if (map->queues[pos] != index)
			continue;
		return map;
	}

	/* Need to add queue to this CPU's existing map */
	if (map) {
		if (pos < map->alloc_len)
			return map;

		alloc_len = map->alloc_len * 2;
	}

	/* Need to allocate new map to store queue on this CPU's map */
	new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
			       cpu_to_node(cpu));
	if (!new_map)
		return NULL;

	for (i = 0; i < pos; i++)
		new_map->queues[i] = map->queues[i];
	new_map->alloc_len = alloc_len;
	new_map->len = pos;

	return new_map;
}

1858 1859
int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask, u16 index)
{
1860
	struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
1861 1862
	struct xps_map *map, *new_map;
	int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
1863 1864
	int cpu, numa_node_id = -2;
	bool active = false;
1865 1866 1867 1868 1869

	mutex_lock(&xps_map_mutex);

	dev_maps = xmap_dereference(dev->xps_maps);

1870 1871 1872 1873 1874 1875 1876
	/* allocate memory for queue storage */
	for_each_online_cpu(cpu) {
		if (!cpumask_test_cpu(cpu, mask))
			continue;

		if (!new_dev_maps)
			new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
1877 1878
		if (!new_dev_maps) {
			mutex_unlock(&xps_map_mutex);
1879
			return -ENOMEM;
1880
		}
1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894

		map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
				 NULL;

		map = expand_xps_map(map, cpu, index);
		if (!map)
			goto error;

		RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
	}

	if (!new_dev_maps)
		goto out_no_new_maps;

1895
	for_each_possible_cpu(cpu) {
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
		if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
			/* add queue to CPU maps */
			int pos = 0;

			map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
			while ((pos < map->len) && (map->queues[pos] != index))
				pos++;

			if (pos == map->len)
				map->queues[map->len++] = index;
1906 1907 1908 1909 1910 1911
#ifdef CONFIG_NUMA
			if (numa_node_id == -2)
				numa_node_id = cpu_to_node(cpu);
			else if (numa_node_id != cpu_to_node(cpu))
				numa_node_id = -1;
#endif
1912 1913 1914 1915
		} else if (dev_maps) {
			/* fill in the new device map from the old device map */
			map = xmap_dereference(dev_maps->cpu_map[cpu]);
			RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1916
		}
1917

1918 1919
	}

1920 1921
	rcu_assign_pointer(dev->xps_maps, new_dev_maps);

1922
	/* Cleanup old maps */
1923 1924 1925 1926 1927 1928 1929
	if (dev_maps) {
		for_each_possible_cpu(cpu) {
			new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
			map = xmap_dereference(dev_maps->cpu_map[cpu]);
			if (map && map != new_map)
				kfree_rcu(map, rcu);
		}
1930

1931
		kfree_rcu(dev_maps, rcu);
1932 1933
	}

1934 1935
	dev_maps = new_dev_maps;
	active = true;
1936

1937 1938
out_no_new_maps:
	/* update Tx queue numa node */
1939 1940 1941 1942
	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
				     (numa_node_id >= 0) ? numa_node_id :
				     NUMA_NO_NODE);

1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
	if (!dev_maps)
		goto out_no_maps;

	/* removes queue from unused CPUs */
	for_each_possible_cpu(cpu) {
		if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
			continue;

		if (remove_xps_queue(dev_maps, cpu, index))
			active = true;
	}

	/* free map if not active */
	if (!active) {
		RCU_INIT_POINTER(dev->xps_maps, NULL);
		kfree_rcu(dev_maps, rcu);
	}

out_no_maps:
1962 1963 1964 1965
	mutex_unlock(&xps_map_mutex);

	return 0;
error:
1966 1967 1968 1969 1970 1971 1972 1973 1974
	/* remove any maps that we added */
	for_each_possible_cpu(cpu) {
		new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
		map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
				 NULL;
		if (new_map && new_map != map)
			kfree(new_map);
	}

1975 1976 1977 1978 1979 1980 1981 1982
	mutex_unlock(&xps_map_mutex);

	kfree(new_dev_maps);
	return -ENOMEM;
}
EXPORT_SYMBOL(netif_set_xps_queue);

#endif
1983 1984 1985 1986
/*
 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
 */
1987
int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
1988
{
T
Tom Herbert 已提交
1989 1990
	int rc;

1991 1992
	if (txq < 1 || txq > dev->num_tx_queues)
		return -EINVAL;
1993

1994 1995
	if (dev->reg_state == NETREG_REGISTERED ||
	    dev->reg_state == NETREG_UNREGISTERING) {
1996 1997
		ASSERT_RTNL();

T
Tom Herbert 已提交
1998 1999
		rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
						  txq);
T
Tom Herbert 已提交
2000 2001 2002
		if (rc)
			return rc;

2003 2004 2005
		if (dev->num_tc)
			netif_setup_tc(dev, txq);

2006
		if (txq < dev->real_num_tx_queues) {
2007
			qdisc_reset_all_tx_gt(dev, txq);
2008 2009 2010 2011
#ifdef CONFIG_XPS
			netif_reset_xps_queues_gt(dev, txq);
#endif
		}
2012
	}
2013 2014 2015

	dev->real_num_tx_queues = txq;
	return 0;
2016 2017
}
EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2018

2019 2020 2021 2022 2023 2024 2025 2026
#ifdef CONFIG_RPS
/**
 *	netif_set_real_num_rx_queues - set actual number of RX queues used
 *	@dev: Network device
 *	@rxq: Actual number of RX queues
 *
 *	This must be called either with the rtnl_lock held or before
 *	registration of the net device.  Returns 0 on success, or a
2027 2028
 *	negative error code.  If called before registration, it always
 *	succeeds.
2029 2030 2031 2032 2033
 */
int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
{
	int rc;

T
Tom Herbert 已提交
2034 2035 2036
	if (rxq < 1 || rxq > dev->num_rx_queues)
		return -EINVAL;

2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051
	if (dev->reg_state == NETREG_REGISTERED) {
		ASSERT_RTNL();

		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
						  rxq);
		if (rc)
			return rc;
	}

	dev->real_num_rx_queues = rxq;
	return 0;
}
EXPORT_SYMBOL(netif_set_real_num_rx_queues);
#endif

2052 2053
/**
 * netif_get_num_default_rss_queues - default number of RSS queues
2054 2055 2056 2057
 *
 * This routine should set an upper limit on the number of RSS queues
 * used by default by multiqueue devices.
 */
2058
int netif_get_num_default_rss_queues(void)
2059 2060 2061 2062 2063
{
	return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
}
EXPORT_SYMBOL(netif_get_num_default_rss_queues);

2064
static inline void __netif_reschedule(struct Qdisc *q)
2065
{
2066 2067
	struct softnet_data *sd;
	unsigned long flags;
2068

2069 2070
	local_irq_save(flags);
	sd = &__get_cpu_var(softnet_data);
2071 2072 2073
	q->next_sched = NULL;
	*sd->output_queue_tailp = q;
	sd->output_queue_tailp = &q->next_sched;
2074 2075 2076 2077 2078 2079 2080 2081
	raise_softirq_irqoff(NET_TX_SOFTIRQ);
	local_irq_restore(flags);
}

void __netif_schedule(struct Qdisc *q)
{
	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
		__netif_reschedule(q);
2082 2083 2084
}
EXPORT_SYMBOL(__netif_schedule);

2085
void dev_kfree_skb_irq(struct sk_buff *skb)
2086
{
2087
	if (atomic_dec_and_test(&skb->users)) {
2088 2089
		struct softnet_data *sd;
		unsigned long flags;
2090

2091 2092 2093 2094 2095 2096 2097
		local_irq_save(flags);
		sd = &__get_cpu_var(softnet_data);
		skb->next = sd->completion_queue;
		sd->completion_queue = skb;
		raise_softirq_irqoff(NET_TX_SOFTIRQ);
		local_irq_restore(flags);
	}
2098
}
2099
EXPORT_SYMBOL(dev_kfree_skb_irq);
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110

void dev_kfree_skb_any(struct sk_buff *skb)
{
	if (in_irq() || irqs_disabled())
		dev_kfree_skb_irq(skb);
	else
		dev_kfree_skb(skb);
}
EXPORT_SYMBOL(dev_kfree_skb_any);


2111 2112 2113 2114 2115 2116
/**
 * netif_device_detach - mark device as removed
 * @dev: network device
 *
 * Mark device as removed from system and therefore no longer available.
 */
2117 2118 2119 2120
void netif_device_detach(struct net_device *dev)
{
	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
	    netif_running(dev)) {
2121
		netif_tx_stop_all_queues(dev);
2122 2123 2124 2125
	}
}
EXPORT_SYMBOL(netif_device_detach);

2126 2127 2128 2129 2130 2131
/**
 * netif_device_attach - mark device as attached
 * @dev: network device
 *
 * Mark device as attached from system and restart if needed.
 */
2132 2133 2134 2135
void netif_device_attach(struct net_device *dev)
{
	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
	    netif_running(dev)) {
2136
		netif_tx_wake_all_queues(dev);
2137
		__netdev_watchdog_up(dev);
2138 2139 2140 2141
	}
}
EXPORT_SYMBOL(netif_device_attach);

2142 2143
static void skb_warn_bad_offload(const struct sk_buff *skb)
{
2144
	static const netdev_features_t null_features = 0;
2145 2146 2147
	struct net_device *dev = skb->dev;
	const char *driver = "";

2148 2149 2150
	if (!net_ratelimit())
		return;

2151 2152 2153 2154 2155
	if (dev && dev->dev.parent)
		driver = dev_driver_string(dev->dev.parent);

	WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
	     "gso_type=%d ip_summed=%d\n",
2156 2157
	     driver, dev ? &dev->features : &null_features,
	     skb->sk ? &skb->sk->sk_route_caps : &null_features,
2158 2159 2160 2161
	     skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
	     skb_shinfo(skb)->gso_type, skb->ip_summed);
}

L
Linus Torvalds 已提交
2162 2163 2164 2165
/*
 * Invalidate hardware checksum when packet is to be mangled, and
 * complete checksum manually on outgoing path.
 */
2166
int skb_checksum_help(struct sk_buff *skb)
L
Linus Torvalds 已提交
2167
{
2168
	__wsum csum;
2169
	int ret = 0, offset;
L
Linus Torvalds 已提交
2170

2171
	if (skb->ip_summed == CHECKSUM_COMPLETE)
2172 2173 2174
		goto out_set_summed;

	if (unlikely(skb_shinfo(skb)->gso_size)) {
2175 2176
		skb_warn_bad_offload(skb);
		return -EINVAL;
L
Linus Torvalds 已提交
2177 2178
	}

2179 2180 2181 2182 2183 2184 2185 2186 2187
	/* Before computing a checksum, we should make sure no frag could
	 * be modified by an external entity : checksum could be wrong.
	 */
	if (skb_has_shared_frag(skb)) {
		ret = __skb_linearize(skb);
		if (ret)
			goto out;
	}

2188
	offset = skb_checksum_start_offset(skb);
2189 2190 2191 2192 2193 2194 2195 2196
	BUG_ON(offset >= skb_headlen(skb));
	csum = skb_checksum(skb, offset, skb->len - offset, 0);

	offset += skb->csum_offset;
	BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));

	if (skb_cloned(skb) &&
	    !skb_clone_writable(skb, offset + sizeof(__sum16))) {
L
Linus Torvalds 已提交
2197 2198 2199 2200 2201
		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
		if (ret)
			goto out;
	}

2202
	*(__sum16 *)(skb->data + offset) = csum_fold(csum);
2203
out_set_summed:
L
Linus Torvalds 已提交
2204
	skb->ip_summed = CHECKSUM_NONE;
2205
out:
L
Linus Torvalds 已提交
2206 2207
	return ret;
}
E
Eric Dumazet 已提交
2208
EXPORT_SYMBOL(skb_checksum_help);
L
Linus Torvalds 已提交
2209

2210
__be16 skb_network_protocol(struct sk_buff *skb)
2211
{
A
Al Viro 已提交
2212
	__be16 type = skb->protocol;
2213
	int vlan_depth = ETH_HLEN;
2214

2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225
	/* Tunnel gso handlers can set protocol to ethernet. */
	if (type == htons(ETH_P_TEB)) {
		struct ethhdr *eth;

		if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
			return 0;

		eth = (struct ethhdr *)skb_mac_header(skb);
		type = eth->h_proto;
	}

P
Patrick McHardy 已提交
2226
	while (type == htons(ETH_P_8021Q) || type == htons(ETH_P_8021AD)) {
2227
		struct vlan_hdr *vh;
2228

2229
		if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
2230
			return 0;
2231

2232 2233 2234
		vh = (struct vlan_hdr *)(skb->data + vlan_depth);
		type = vh->h_vlan_encapsulated_proto;
		vlan_depth += VLAN_HLEN;
2235 2236
	}

2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254
	return type;
}

/**
 *	skb_mac_gso_segment - mac layer segmentation handler.
 *	@skb: buffer to segment
 *	@features: features for the output path (see dev->features)
 */
struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
				    netdev_features_t features)
{
	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
	struct packet_offload *ptype;
	__be16 type = skb_network_protocol(skb);

	if (unlikely(!type))
		return ERR_PTR(-EINVAL);

2255 2256 2257
	__skb_pull(skb, skb->mac_len);

	rcu_read_lock();
2258
	list_for_each_entry_rcu(ptype, &offload_base, list) {
2259
		if (ptype->type == type && ptype->callbacks.gso_segment) {
2260
			if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
2261 2262
				int err;

2263
				err = ptype->callbacks.gso_send_check(skb);
2264 2265 2266
				segs = ERR_PTR(err);
				if (err || skb_gso_ok(skb, features))
					break;
2267 2268
				__skb_push(skb, (skb->data -
						 skb_network_header(skb)));
2269
			}
2270
			segs = ptype->callbacks.gso_segment(skb, features);
2271 2272 2273 2274 2275
			break;
		}
	}
	rcu_read_unlock();

2276
	__skb_push(skb, skb->data - skb_mac_header(skb));
2277

2278 2279
	return segs;
}
2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316
EXPORT_SYMBOL(skb_mac_gso_segment);


/* openvswitch calls this on rx path, so we need a different check.
 */
static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
{
	if (tx_path)
		return skb->ip_summed != CHECKSUM_PARTIAL;
	else
		return skb->ip_summed == CHECKSUM_NONE;
}

/**
 *	__skb_gso_segment - Perform segmentation on skb.
 *	@skb: buffer to segment
 *	@features: features for the output path (see dev->features)
 *	@tx_path: whether it is called in TX path
 *
 *	This function segments the given skb and returns a list of segments.
 *
 *	It may return NULL if the skb requires no segmentation.  This is
 *	only possible when GSO is used for verifying header integrity.
 */
struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
				  netdev_features_t features, bool tx_path)
{
	if (unlikely(skb_needs_check(skb, tx_path))) {
		int err;

		skb_warn_bad_offload(skb);

		if (skb_header_cloned(skb) &&
		    (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
			return ERR_PTR(err);
	}

2317
	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2318 2319 2320 2321 2322
	skb_reset_mac_header(skb);
	skb_reset_mac_len(skb);

	return skb_mac_gso_segment(skb, features);
}
2323
EXPORT_SYMBOL(__skb_gso_segment);
2324

2325 2326 2327 2328 2329
/* Take action when hardware reception checksum errors are detected. */
#ifdef CONFIG_BUG
void netdev_rx_csum_fault(struct net_device *dev)
{
	if (net_ratelimit()) {
2330
		pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2331 2332 2333 2334 2335 2336
		dump_stack();
	}
}
EXPORT_SYMBOL(netdev_rx_csum_fault);
#endif

L
Linus Torvalds 已提交
2337 2338 2339 2340 2341
/* Actually, we should eliminate this check as soon as we know, that:
 * 1. IOMMU is present and allows to map all the memory.
 * 2. No high memory really exists on this machine.
 */

E
Eric Dumazet 已提交
2342
static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
L
Linus Torvalds 已提交
2343
{
2344
#ifdef CONFIG_HIGHMEM
L
Linus Torvalds 已提交
2345
	int i;
2346
	if (!(dev->features & NETIF_F_HIGHDMA)) {
2347 2348 2349
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
			if (PageHighMem(skb_frag_page(frag)))
2350
				return 1;
2351
		}
2352
	}
L
Linus Torvalds 已提交
2353

2354 2355
	if (PCI_DMA_BUS_IS_PHYS) {
		struct device *pdev = dev->dev.parent;
L
Linus Torvalds 已提交
2356

E
Eric Dumazet 已提交
2357 2358
		if (!pdev)
			return 0;
2359
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2360 2361
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
			dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2362 2363 2364 2365
			if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
				return 1;
		}
	}
2366
#endif
L
Linus Torvalds 已提交
2367 2368 2369
	return 0;
}

2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395
struct dev_gso_cb {
	void (*destructor)(struct sk_buff *skb);
};

#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)

static void dev_gso_skb_destructor(struct sk_buff *skb)
{
	struct dev_gso_cb *cb;

	do {
		struct sk_buff *nskb = skb->next;

		skb->next = nskb->next;
		nskb->next = NULL;
		kfree_skb(nskb);
	} while (skb->next);

	cb = DEV_GSO_CB(skb);
	if (cb->destructor)
		cb->destructor(skb);
}

/**
 *	dev_gso_segment - Perform emulated hardware segmentation on skb.
 *	@skb: buffer to segment
2396
 *	@features: device features as applicable to this skb
2397 2398 2399 2400
 *
 *	This function segments the given skb and stores the list of segments
 *	in skb->next.
 */
2401
static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
2402 2403
{
	struct sk_buff *segs;
2404 2405 2406 2407 2408 2409

	segs = skb_gso_segment(skb, features);

	/* Verifying header integrity only. */
	if (!segs)
		return 0;
2410

2411
	if (IS_ERR(segs))
2412 2413 2414 2415 2416 2417 2418 2419 2420
		return PTR_ERR(segs);

	skb->next = segs;
	DEV_GSO_CB(skb)->destructor = skb->destructor;
	skb->destructor = dev_gso_skb_destructor;

	return 0;
}

2421 2422
static netdev_features_t harmonize_features(struct sk_buff *skb,
	__be16 protocol, netdev_features_t features)
2423
{
2424 2425
	if (skb->ip_summed != CHECKSUM_NONE &&
	    !can_checksum_protocol(features, protocol)) {
2426 2427 2428 2429 2430 2431 2432 2433
		features &= ~NETIF_F_ALL_CSUM;
	} else if (illegal_highdma(skb->dev, skb)) {
		features &= ~NETIF_F_SG;
	}

	return features;
}

2434
netdev_features_t netif_skb_features(struct sk_buff *skb)
2435 2436
{
	__be16 protocol = skb->protocol;
2437
	netdev_features_t features = skb->dev->features;
2438

2439 2440 2441
	if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
		features &= ~NETIF_F_GSO_MASK;

P
Patrick McHardy 已提交
2442
	if (protocol == htons(ETH_P_8021Q) || protocol == htons(ETH_P_8021AD)) {
2443 2444
		struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
		protocol = veh->h_vlan_encapsulated_proto;
2445 2446 2447
	} else if (!vlan_tx_tag_present(skb)) {
		return harmonize_features(skb, protocol, features);
	}
2448

P
Patrick McHardy 已提交
2449 2450
	features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_CTAG_TX |
					       NETIF_F_HW_VLAN_STAG_TX);
2451

P
Patrick McHardy 已提交
2452
	if (protocol != htons(ETH_P_8021Q) && protocol != htons(ETH_P_8021AD)) {
2453 2454 2455
		return harmonize_features(skb, protocol, features);
	} else {
		features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
P
Patrick McHardy 已提交
2456 2457
				NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
				NETIF_F_HW_VLAN_STAG_TX;
2458 2459
		return harmonize_features(skb, protocol, features);
	}
2460
}
2461
EXPORT_SYMBOL(netif_skb_features);
2462

2463 2464 2465
/*
 * Returns true if either:
 *	1. skb has frag_list and the device doesn't support FRAGLIST, or
2466
 *	2. skb is fragmented and the device does not support SG.
2467 2468
 */
static inline int skb_needs_linearize(struct sk_buff *skb,
2469
				      netdev_features_t features)
2470
{
2471 2472 2473
	return skb_is_nonlinear(skb) &&
			((skb_has_frag_list(skb) &&
				!(features & NETIF_F_FRAGLIST)) ||
2474
			(skb_shinfo(skb)->nr_frags &&
2475
				!(features & NETIF_F_SG)));
2476 2477
}

2478 2479
int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
			struct netdev_queue *txq)
2480
{
2481
	const struct net_device_ops *ops = dev->netdev_ops;
2482
	int rc = NETDEV_TX_OK;
2483
	unsigned int skb_len;
2484

2485
	if (likely(!skb->next)) {
2486
		netdev_features_t features;
2487

2488
		/*
L
Lucas De Marchi 已提交
2489
		 * If device doesn't need skb->dst, release it right now while
2490 2491
		 * its hot in this cpu cache
		 */
E
Eric Dumazet 已提交
2492 2493 2494
		if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
			skb_dst_drop(skb);

2495 2496
		features = netif_skb_features(skb);

2497
		if (vlan_tx_tag_present(skb) &&
2498 2499 2500
		    !vlan_hw_offload_capable(features, skb->vlan_proto)) {
			skb = __vlan_put_tag(skb, skb->vlan_proto,
					     vlan_tx_tag_get(skb));
2501 2502 2503 2504 2505 2506
			if (unlikely(!skb))
				goto out;

			skb->vlan_tci = 0;
		}

2507 2508 2509 2510 2511 2512 2513
		/* If encapsulation offload request, verify we are testing
		 * hardware encapsulation features instead of standard
		 * features for the netdev
		 */
		if (skb->encapsulation)
			features &= dev->hw_enc_features;

2514
		if (netif_needs_gso(skb, features)) {
2515
			if (unlikely(dev_gso_segment(skb, features)))
2516 2517 2518
				goto out_kfree_skb;
			if (skb->next)
				goto gso;
2519
		} else {
2520
			if (skb_needs_linearize(skb, features) &&
2521 2522 2523 2524 2525 2526 2527 2528
			    __skb_linearize(skb))
				goto out_kfree_skb;

			/* If packet is not checksummed and device does not
			 * support checksumming for this protocol, complete
			 * checksumming here.
			 */
			if (skb->ip_summed == CHECKSUM_PARTIAL) {
2529 2530 2531 2532 2533 2534
				if (skb->encapsulation)
					skb_set_inner_transport_header(skb,
						skb_checksum_start_offset(skb));
				else
					skb_set_transport_header(skb,
						skb_checksum_start_offset(skb));
2535
				if (!(features & NETIF_F_ALL_CSUM) &&
2536 2537 2538
				     skb_checksum_help(skb))
					goto out_kfree_skb;
			}
2539 2540
		}

2541 2542 2543
		if (!list_empty(&ptype_all))
			dev_queue_xmit_nit(skb, dev);

2544
		skb_len = skb->len;
2545
		rc = ops->ndo_start_xmit(skb, dev);
2546
		trace_net_dev_xmit(skb, rc, dev, skb_len);
2547
		if (rc == NETDEV_TX_OK)
E
Eric Dumazet 已提交
2548
			txq_trans_update(txq);
2549
		return rc;
2550 2551
	}

2552
gso:
2553 2554 2555 2556 2557
	do {
		struct sk_buff *nskb = skb->next;

		skb->next = nskb->next;
		nskb->next = NULL;
2558

2559 2560 2561
		if (!list_empty(&ptype_all))
			dev_queue_xmit_nit(nskb, dev);

2562
		skb_len = nskb->len;
2563
		rc = ops->ndo_start_xmit(nskb, dev);
2564
		trace_net_dev_xmit(nskb, rc, dev, skb_len);
2565
		if (unlikely(rc != NETDEV_TX_OK)) {
2566 2567
			if (rc & ~NETDEV_TX_MASK)
				goto out_kfree_gso_skb;
2568
			nskb->next = skb->next;
2569 2570 2571
			skb->next = nskb;
			return rc;
		}
E
Eric Dumazet 已提交
2572
		txq_trans_update(txq);
2573
		if (unlikely(netif_xmit_stopped(txq) && skb->next))
2574
			return NETDEV_TX_BUSY;
2575
	} while (skb->next);
2576

2577
out_kfree_gso_skb:
2578
	if (likely(skb->next == NULL)) {
2579
		skb->destructor = DEV_GSO_CB(skb)->destructor;
2580 2581 2582
		consume_skb(skb);
		return rc;
	}
2583 2584
out_kfree_skb:
	kfree_skb(skb);
2585
out:
2586
	return rc;
2587 2588
}

2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
static void qdisc_pkt_len_init(struct sk_buff *skb)
{
	const struct skb_shared_info *shinfo = skb_shinfo(skb);

	qdisc_skb_cb(skb)->pkt_len = skb->len;

	/* To get more precise estimation of bytes sent on wire,
	 * we add to pkt_len the headers size of all segments
	 */
	if (shinfo->gso_size)  {
2599
		unsigned int hdr_len;
2600
		u16 gso_segs = shinfo->gso_segs;
2601

2602 2603 2604 2605
		/* mac layer + network layer */
		hdr_len = skb_transport_header(skb) - skb_mac_header(skb);

		/* + transport layer */
2606 2607 2608 2609
		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
			hdr_len += tcp_hdrlen(skb);
		else
			hdr_len += sizeof(struct udphdr);
2610 2611 2612 2613 2614 2615

		if (shinfo->gso_type & SKB_GSO_DODGY)
			gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
						shinfo->gso_size);

		qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
2616 2617 2618
	}
}

2619 2620 2621 2622 2623
static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
				 struct net_device *dev,
				 struct netdev_queue *txq)
{
	spinlock_t *root_lock = qdisc_lock(q);
E
Eric Dumazet 已提交
2624
	bool contended;
2625 2626
	int rc;

2627
	qdisc_pkt_len_init(skb);
E
Eric Dumazet 已提交
2628
	qdisc_calculate_pkt_len(skb, q);
2629 2630 2631 2632 2633 2634
	/*
	 * Heuristic to force contended enqueues to serialize on a
	 * separate lock before trying to get qdisc main lock.
	 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
	 * and dequeue packets faster.
	 */
E
Eric Dumazet 已提交
2635
	contended = qdisc_is_running(q);
2636 2637 2638
	if (unlikely(contended))
		spin_lock(&q->busylock);

2639 2640 2641 2642 2643
	spin_lock(root_lock);
	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
		kfree_skb(skb);
		rc = NET_XMIT_DROP;
	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2644
		   qdisc_run_begin(q)) {
2645 2646 2647 2648 2649
		/*
		 * This is a work-conserving queue; there are no old skbs
		 * waiting to be sent out; and the qdisc is not running -
		 * xmit the skb directly.
		 */
E
Eric Dumazet 已提交
2650 2651
		if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
			skb_dst_force(skb);
2652 2653 2654

		qdisc_bstats_update(q, skb);

2655 2656 2657 2658 2659
		if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
			if (unlikely(contended)) {
				spin_unlock(&q->busylock);
				contended = false;
			}
2660
			__qdisc_run(q);
2661
		} else
2662
			qdisc_run_end(q);
2663 2664 2665

		rc = NET_XMIT_SUCCESS;
	} else {
E
Eric Dumazet 已提交
2666
		skb_dst_force(skb);
E
Eric Dumazet 已提交
2667
		rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2668 2669 2670 2671 2672 2673 2674
		if (qdisc_run_begin(q)) {
			if (unlikely(contended)) {
				spin_unlock(&q->busylock);
				contended = false;
			}
			__qdisc_run(q);
		}
2675 2676
	}
	spin_unlock(root_lock);
2677 2678
	if (unlikely(contended))
		spin_unlock(&q->busylock);
2679 2680 2681
	return rc;
}

2682 2683 2684
#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
static void skb_update_prio(struct sk_buff *skb)
{
I
Igor Maravic 已提交
2685
	struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
2686

2687 2688 2689 2690 2691 2692
	if (!skb->priority && skb->sk && map) {
		unsigned int prioidx = skb->sk->sk_cgrp_prioidx;

		if (prioidx < map->priomap_len)
			skb->priority = map->priomap[prioidx];
	}
2693 2694 2695 2696 2697
}
#else
#define skb_update_prio(skb)
#endif

2698
static DEFINE_PER_CPU(int, xmit_recursion);
2699
#define RECURSION_LIMIT 10
2700

2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717
/**
 *	dev_loopback_xmit - loop back @skb
 *	@skb: buffer to transmit
 */
int dev_loopback_xmit(struct sk_buff *skb)
{
	skb_reset_mac_header(skb);
	__skb_pull(skb, skb_network_offset(skb));
	skb->pkt_type = PACKET_LOOPBACK;
	skb->ip_summed = CHECKSUM_UNNECESSARY;
	WARN_ON(!skb_dst(skb));
	skb_dst_force(skb);
	netif_rx_ni(skb);
	return 0;
}
EXPORT_SYMBOL(dev_loopback_xmit);

2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742
/**
 *	dev_queue_xmit - transmit a buffer
 *	@skb: buffer to transmit
 *
 *	Queue a buffer for transmission to a network device. The caller must
 *	have set the device and priority and built the buffer before calling
 *	this function. The function can be called from an interrupt.
 *
 *	A negative errno code is returned on a failure. A success does not
 *	guarantee the frame will be transmitted as it may be dropped due
 *	to congestion or traffic shaping.
 *
 * -----------------------------------------------------------------------------------
 *      I notice this method can also return errors from the queue disciplines,
 *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
 *      be positive.
 *
 *      Regardless of the return value, the skb is consumed, so it is currently
 *      difficult to retry a send to this method.  (You can bump the ref count
 *      before sending to hold a reference for retry if you are careful.)
 *
 *      When calling this method, interrupts MUST be enabled.  This is because
 *      the BH enable code must have IRQs enabled so that it will not deadlock.
 *          --BLG
 */
L
Linus Torvalds 已提交
2743 2744 2745
int dev_queue_xmit(struct sk_buff *skb)
{
	struct net_device *dev = skb->dev;
2746
	struct netdev_queue *txq;
L
Linus Torvalds 已提交
2747 2748 2749
	struct Qdisc *q;
	int rc = -ENOMEM;

2750 2751
	skb_reset_mac_header(skb);

2752 2753
	/* Disable soft irqs for various locks below. Also
	 * stops preemption for RCU.
L
Linus Torvalds 已提交
2754
	 */
2755
	rcu_read_lock_bh();
L
Linus Torvalds 已提交
2756

2757 2758
	skb_update_prio(skb);

2759
	txq = netdev_pick_tx(dev, skb);
2760
	q = rcu_dereference_bh(txq->qdisc);
2761

L
Linus Torvalds 已提交
2762
#ifdef CONFIG_NET_CLS_ACT
E
Eric Dumazet 已提交
2763
	skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
L
Linus Torvalds 已提交
2764
#endif
2765
	trace_net_dev_queue(skb);
L
Linus Torvalds 已提交
2766
	if (q->enqueue) {
2767
		rc = __dev_xmit_skb(skb, q, dev, txq);
2768
		goto out;
L
Linus Torvalds 已提交
2769 2770 2771 2772 2773
	}

	/* The device has no queue. Common case for software devices:
	   loopback, all the sorts of tunnels...

H
Herbert Xu 已提交
2774 2775
	   Really, it is unlikely that netif_tx_lock protection is necessary
	   here.  (f.e. loopback and IP tunnels are clean ignoring statistics
L
Linus Torvalds 已提交
2776 2777 2778 2779 2780 2781 2782 2783 2784 2785
	   counters.)
	   However, it is possible, that they rely on protection
	   made by us here.

	   Check this and shot the lock. It is not prone from deadlocks.
	   Either shot noqueue qdisc, it is even simpler 8)
	 */
	if (dev->flags & IFF_UP) {
		int cpu = smp_processor_id(); /* ok because BHs are off */

2786
		if (txq->xmit_lock_owner != cpu) {
L
Linus Torvalds 已提交
2787

2788 2789 2790
			if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
				goto recursion_alert;

2791
			HARD_TX_LOCK(dev, txq, cpu);
L
Linus Torvalds 已提交
2792

2793
			if (!netif_xmit_stopped(txq)) {
2794
				__this_cpu_inc(xmit_recursion);
2795
				rc = dev_hard_start_xmit(skb, dev, txq);
2796
				__this_cpu_dec(xmit_recursion);
2797
				if (dev_xmit_complete(rc)) {
2798
					HARD_TX_UNLOCK(dev, txq);
L
Linus Torvalds 已提交
2799 2800 2801
					goto out;
				}
			}
2802
			HARD_TX_UNLOCK(dev, txq);
2803 2804
			net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
					     dev->name);
L
Linus Torvalds 已提交
2805 2806
		} else {
			/* Recursion is detected! It is possible,
2807 2808 2809
			 * unfortunately
			 */
recursion_alert:
2810 2811
			net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
					     dev->name);
L
Linus Torvalds 已提交
2812 2813 2814 2815
		}
	}

	rc = -ENETDOWN;
2816
	rcu_read_unlock_bh();
L
Linus Torvalds 已提交
2817 2818 2819 2820

	kfree_skb(skb);
	return rc;
out:
2821
	rcu_read_unlock_bh();
L
Linus Torvalds 已提交
2822 2823
	return rc;
}
E
Eric Dumazet 已提交
2824
EXPORT_SYMBOL(dev_queue_xmit);
L
Linus Torvalds 已提交
2825 2826 2827 2828 2829 2830


/*=======================================================================
			Receiver routines
  =======================================================================*/

2831
int netdev_max_backlog __read_mostly = 1000;
E
Eric Dumazet 已提交
2832 2833
EXPORT_SYMBOL(netdev_max_backlog);

E
Eric Dumazet 已提交
2834
int netdev_tstamp_prequeue __read_mostly = 1;
2835 2836
int netdev_budget __read_mostly = 300;
int weight_p __read_mostly = 64;            /* old backlog weight */
L
Linus Torvalds 已提交
2837

E
Eric Dumazet 已提交
2838 2839 2840 2841 2842 2843 2844 2845
/* Called with irq disabled */
static inline void ____napi_schedule(struct softnet_data *sd,
				     struct napi_struct *napi)
{
	list_add_tail(&napi->poll_list, &sd->poll_list);
	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
}

2846 2847 2848
#ifdef CONFIG_RPS

/* One global table that all flow-based protocols share. */
E
Eric Dumazet 已提交
2849
struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2850 2851
EXPORT_SYMBOL(rps_sock_flow_table);

2852
struct static_key rps_needed __read_mostly;
2853

2854 2855 2856 2857
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
	    struct rps_dev_flow *rflow, u16 next_cpu)
{
2858
	if (next_cpu != RPS_NO_CPU) {
2859 2860 2861 2862 2863 2864 2865 2866 2867
#ifdef CONFIG_RFS_ACCEL
		struct netdev_rx_queue *rxqueue;
		struct rps_dev_flow_table *flow_table;
		struct rps_dev_flow *old_rflow;
		u32 flow_id;
		u16 rxq_index;
		int rc;

		/* Should we steer this flow to a different hardware queue? */
2868 2869
		if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
		    !(dev->features & NETIF_F_NTUPLE))
2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891
			goto out;
		rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
		if (rxq_index == skb_get_rx_queue(skb))
			goto out;

		rxqueue = dev->_rx + rxq_index;
		flow_table = rcu_dereference(rxqueue->rps_flow_table);
		if (!flow_table)
			goto out;
		flow_id = skb->rxhash & flow_table->mask;
		rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
							rxq_index, flow_id);
		if (rc < 0)
			goto out;
		old_rflow = rflow;
		rflow = &flow_table->flows[flow_id];
		rflow->filter = rc;
		if (old_rflow->filter == rflow->filter)
			old_rflow->filter = RPS_NO_FILTER;
	out:
#endif
		rflow->last_qtail =
2892
			per_cpu(softnet_data, next_cpu).input_queue_head;
2893 2894
	}

2895
	rflow->cpu = next_cpu;
2896 2897 2898
	return rflow;
}

2899 2900 2901 2902 2903 2904 2905 2906 2907
/*
 * get_rps_cpu is called from netif_receive_skb and returns the target
 * CPU from the RPS map of the receiving queue for a given skb.
 * rcu_read_lock must be held on entry.
 */
static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
		       struct rps_dev_flow **rflowp)
{
	struct netdev_rx_queue *rxqueue;
E
Eric Dumazet 已提交
2908
	struct rps_map *map;
2909 2910 2911 2912 2913 2914 2915
	struct rps_dev_flow_table *flow_table;
	struct rps_sock_flow_table *sock_flow_table;
	int cpu = -1;
	u16 tcpu;

	if (skb_rx_queue_recorded(skb)) {
		u16 index = skb_get_rx_queue(skb);
2916 2917 2918 2919 2920
		if (unlikely(index >= dev->real_num_rx_queues)) {
			WARN_ONCE(dev->real_num_rx_queues > 1,
				  "%s received packet on queue %u, but number "
				  "of RX queues is %u\n",
				  dev->name, index, dev->real_num_rx_queues);
2921 2922 2923 2924 2925 2926
			goto done;
		}
		rxqueue = dev->_rx + index;
	} else
		rxqueue = dev->_rx;

E
Eric Dumazet 已提交
2927 2928
	map = rcu_dereference(rxqueue->rps_map);
	if (map) {
2929
		if (map->len == 1 &&
2930
		    !rcu_access_pointer(rxqueue->rps_flow_table)) {
2931 2932 2933 2934 2935
			tcpu = map->cpus[0];
			if (cpu_online(tcpu))
				cpu = tcpu;
			goto done;
		}
2936
	} else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
2937
		goto done;
2938
	}
2939

2940
	skb_reset_network_header(skb);
2941 2942 2943
	if (!skb_get_rxhash(skb))
		goto done;

T
Tom Herbert 已提交
2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969
	flow_table = rcu_dereference(rxqueue->rps_flow_table);
	sock_flow_table = rcu_dereference(rps_sock_flow_table);
	if (flow_table && sock_flow_table) {
		u16 next_cpu;
		struct rps_dev_flow *rflow;

		rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
		tcpu = rflow->cpu;

		next_cpu = sock_flow_table->ents[skb->rxhash &
		    sock_flow_table->mask];

		/*
		 * If the desired CPU (where last recvmsg was done) is
		 * different from current CPU (one in the rx-queue flow
		 * table entry), switch if one of the following holds:
		 *   - Current CPU is unset (equal to RPS_NO_CPU).
		 *   - Current CPU is offline.
		 *   - The current CPU's queue tail has advanced beyond the
		 *     last packet that was enqueued using this table entry.
		 *     This guarantees that all previous packets for the flow
		 *     have been dequeued, thus preserving in order delivery.
		 */
		if (unlikely(tcpu != next_cpu) &&
		    (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
2970 2971
		      rflow->last_qtail)) >= 0)) {
			tcpu = next_cpu;
2972
			rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
2973
		}
2974

T
Tom Herbert 已提交
2975 2976 2977 2978 2979 2980 2981
		if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
			*rflowp = rflow;
			cpu = tcpu;
			goto done;
		}
	}

T
Tom Herbert 已提交
2982
	if (map) {
T
Tom Herbert 已提交
2983
		tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
T
Tom Herbert 已提交
2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994

		if (cpu_online(tcpu)) {
			cpu = tcpu;
			goto done;
		}
	}

done:
	return cpu;
}

2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034
#ifdef CONFIG_RFS_ACCEL

/**
 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
 * @dev: Device on which the filter was set
 * @rxq_index: RX queue index
 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
 *
 * Drivers that implement ndo_rx_flow_steer() should periodically call
 * this function for each installed filter and remove the filters for
 * which it returns %true.
 */
bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
			 u32 flow_id, u16 filter_id)
{
	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
	struct rps_dev_flow_table *flow_table;
	struct rps_dev_flow *rflow;
	bool expire = true;
	int cpu;

	rcu_read_lock();
	flow_table = rcu_dereference(rxqueue->rps_flow_table);
	if (flow_table && flow_id <= flow_table->mask) {
		rflow = &flow_table->flows[flow_id];
		cpu = ACCESS_ONCE(rflow->cpu);
		if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
		    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
			   rflow->last_qtail) <
		     (int)(10 * flow_table->mask)))
			expire = false;
	}
	rcu_read_unlock();
	return expire;
}
EXPORT_SYMBOL(rps_may_expire_flow);

#endif /* CONFIG_RFS_ACCEL */

T
Tom Herbert 已提交
3035
/* Called from hardirq (IPI) context */
E
Eric Dumazet 已提交
3036
static void rps_trigger_softirq(void *data)
T
Tom Herbert 已提交
3037
{
E
Eric Dumazet 已提交
3038 3039
	struct softnet_data *sd = data;

E
Eric Dumazet 已提交
3040
	____napi_schedule(sd, &sd->backlog);
C
Changli Gao 已提交
3041
	sd->received_rps++;
T
Tom Herbert 已提交
3042
}
E
Eric Dumazet 已提交
3043

T
Tom Herbert 已提交
3044
#endif /* CONFIG_RPS */
T
Tom Herbert 已提交
3045

E
Eric Dumazet 已提交
3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066
/*
 * Check if this softnet_data structure is another cpu one
 * If yes, queue it to our IPI list and return 1
 * If no, return 0
 */
static int rps_ipi_queued(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
	struct softnet_data *mysd = &__get_cpu_var(softnet_data);

	if (sd != mysd) {
		sd->rps_ipi_next = mysd->rps_ipi_list;
		mysd->rps_ipi_list = sd;

		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
		return 1;
	}
#endif /* CONFIG_RPS */
	return 0;
}

3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106
#ifdef CONFIG_NET_FLOW_LIMIT
int netdev_flow_limit_table_len __read_mostly = (1 << 12);
#endif

static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
{
#ifdef CONFIG_NET_FLOW_LIMIT
	struct sd_flow_limit *fl;
	struct softnet_data *sd;
	unsigned int old_flow, new_flow;

	if (qlen < (netdev_max_backlog >> 1))
		return false;

	sd = &__get_cpu_var(softnet_data);

	rcu_read_lock();
	fl = rcu_dereference(sd->flow_limit);
	if (fl) {
		new_flow = skb_get_rxhash(skb) & (fl->num_buckets - 1);
		old_flow = fl->history[fl->history_head];
		fl->history[fl->history_head] = new_flow;

		fl->history_head++;
		fl->history_head &= FLOW_LIMIT_HISTORY - 1;

		if (likely(fl->buckets[old_flow]))
			fl->buckets[old_flow]--;

		if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
			fl->count++;
			rcu_read_unlock();
			return true;
		}
	}
	rcu_read_unlock();
#endif
	return false;
}

T
Tom Herbert 已提交
3107 3108 3109 3110
/*
 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
 * queue (may be a remote CPU queue).
 */
T
Tom Herbert 已提交
3111 3112
static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
			      unsigned int *qtail)
T
Tom Herbert 已提交
3113
{
E
Eric Dumazet 已提交
3114
	struct softnet_data *sd;
T
Tom Herbert 已提交
3115
	unsigned long flags;
3116
	unsigned int qlen;
T
Tom Herbert 已提交
3117

E
Eric Dumazet 已提交
3118
	sd = &per_cpu(softnet_data, cpu);
T
Tom Herbert 已提交
3119 3120 3121

	local_irq_save(flags);

E
Eric Dumazet 已提交
3122
	rps_lock(sd);
3123 3124
	qlen = skb_queue_len(&sd->input_pkt_queue);
	if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3125
		if (skb_queue_len(&sd->input_pkt_queue)) {
T
Tom Herbert 已提交
3126
enqueue:
E
Eric Dumazet 已提交
3127
			__skb_queue_tail(&sd->input_pkt_queue, skb);
3128
			input_queue_tail_incr_save(sd, qtail);
E
Eric Dumazet 已提交
3129
			rps_unlock(sd);
3130
			local_irq_restore(flags);
T
Tom Herbert 已提交
3131 3132 3133
			return NET_RX_SUCCESS;
		}

3134 3135 3136 3137
		/* Schedule NAPI for backlog device
		 * We can use non atomic operation since we own the queue lock
		 */
		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
E
Eric Dumazet 已提交
3138
			if (!rps_ipi_queued(sd))
E
Eric Dumazet 已提交
3139
				____napi_schedule(sd, &sd->backlog);
T
Tom Herbert 已提交
3140 3141 3142 3143
		}
		goto enqueue;
	}

C
Changli Gao 已提交
3144
	sd->dropped++;
E
Eric Dumazet 已提交
3145
	rps_unlock(sd);
T
Tom Herbert 已提交
3146 3147 3148

	local_irq_restore(flags);

3149
	atomic_long_inc(&skb->dev->rx_dropped);
T
Tom Herbert 已提交
3150 3151 3152
	kfree_skb(skb);
	return NET_RX_DROP;
}
L
Linus Torvalds 已提交
3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170

/**
 *	netif_rx	-	post buffer to the network code
 *	@skb: buffer to post
 *
 *	This function receives a packet from a device driver and queues it for
 *	the upper (protocol) levels to process.  It always succeeds. The buffer
 *	may be dropped during processing for congestion control or by the
 *	protocol layers.
 *
 *	return values:
 *	NET_RX_SUCCESS	(no congestion)
 *	NET_RX_DROP     (packet was dropped)
 *
 */

int netif_rx(struct sk_buff *skb)
{
3171
	int ret;
L
Linus Torvalds 已提交
3172 3173 3174 3175 3176

	/* if netpoll wants it, pretend we never saw it */
	if (netpoll_rx(skb))
		return NET_RX_DROP;

3177
	net_timestamp_check(netdev_tstamp_prequeue, skb);
L
Linus Torvalds 已提交
3178

3179
	trace_netif_rx(skb);
E
Eric Dumazet 已提交
3180
#ifdef CONFIG_RPS
3181
	if (static_key_false(&rps_needed)) {
T
Tom Herbert 已提交
3182
		struct rps_dev_flow voidflow, *rflow = &voidflow;
3183 3184
		int cpu;

3185
		preempt_disable();
3186
		rcu_read_lock();
T
Tom Herbert 已提交
3187 3188

		cpu = get_rps_cpu(skb->dev, skb, &rflow);
3189 3190
		if (cpu < 0)
			cpu = smp_processor_id();
T
Tom Herbert 已提交
3191 3192 3193

		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);

3194
		rcu_read_unlock();
3195
		preempt_enable();
3196 3197
	} else
#endif
T
Tom Herbert 已提交
3198 3199 3200 3201 3202
	{
		unsigned int qtail;
		ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
		put_cpu();
	}
3203
	return ret;
L
Linus Torvalds 已提交
3204
}
E
Eric Dumazet 已提交
3205
EXPORT_SYMBOL(netif_rx);
L
Linus Torvalds 已提交
3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236

int netif_rx_ni(struct sk_buff *skb)
{
	int err;

	preempt_disable();
	err = netif_rx(skb);
	if (local_softirq_pending())
		do_softirq();
	preempt_enable();

	return err;
}
EXPORT_SYMBOL(netif_rx_ni);

static void net_tx_action(struct softirq_action *h)
{
	struct softnet_data *sd = &__get_cpu_var(softnet_data);

	if (sd->completion_queue) {
		struct sk_buff *clist;

		local_irq_disable();
		clist = sd->completion_queue;
		sd->completion_queue = NULL;
		local_irq_enable();

		while (clist) {
			struct sk_buff *skb = clist;
			clist = clist->next;

3237
			WARN_ON(atomic_read(&skb->users));
3238
			trace_kfree_skb(skb, net_tx_action);
L
Linus Torvalds 已提交
3239 3240 3241 3242 3243
			__kfree_skb(skb);
		}
	}

	if (sd->output_queue) {
3244
		struct Qdisc *head;
L
Linus Torvalds 已提交
3245 3246 3247 3248

		local_irq_disable();
		head = sd->output_queue;
		sd->output_queue = NULL;
3249
		sd->output_queue_tailp = &sd->output_queue;
L
Linus Torvalds 已提交
3250 3251 3252
		local_irq_enable();

		while (head) {
3253 3254 3255
			struct Qdisc *q = head;
			spinlock_t *root_lock;

L
Linus Torvalds 已提交
3256 3257
			head = head->next_sched;

3258
			root_lock = qdisc_lock(q);
3259
			if (spin_trylock(root_lock)) {
3260 3261 3262
				smp_mb__before_clear_bit();
				clear_bit(__QDISC_STATE_SCHED,
					  &q->state);
3263 3264
				qdisc_run(q);
				spin_unlock(root_lock);
L
Linus Torvalds 已提交
3265
			} else {
3266
				if (!test_bit(__QDISC_STATE_DEACTIVATED,
3267
					      &q->state)) {
3268
					__netif_reschedule(q);
3269 3270 3271 3272 3273
				} else {
					smp_mb__before_clear_bit();
					clear_bit(__QDISC_STATE_SCHED,
						  &q->state);
				}
L
Linus Torvalds 已提交
3274 3275 3276 3277 3278
			}
		}
	}
}

3279 3280
#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
    (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3281 3282 3283
/* This hook is defined here for ATM LANE */
int (*br_fdb_test_addr_hook)(struct net_device *dev,
			     unsigned char *addr) __read_mostly;
3284
EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3285
#endif
L
Linus Torvalds 已提交
3286 3287 3288 3289 3290 3291

#ifdef CONFIG_NET_CLS_ACT
/* TODO: Maybe we should just force sch_ingress to be compiled in
 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
 * a compare and 2 stores extra right now if we dont have it on
 * but have CONFIG_NET_CLS_ACT
L
Lucas De Marchi 已提交
3292 3293
 * NOTE: This doesn't stop any functionality; if you dont have
 * the ingress scheduler, you just can't add policies on ingress.
L
Linus Torvalds 已提交
3294 3295
 *
 */
3296
static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
L
Linus Torvalds 已提交
3297 3298
{
	struct net_device *dev = skb->dev;
3299
	u32 ttl = G_TC_RTTL(skb->tc_verd);
3300 3301
	int result = TC_ACT_OK;
	struct Qdisc *q;
3302

3303
	if (unlikely(MAX_RED_LOOP < ttl++)) {
3304 3305
		net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
				     skb->skb_iif, dev->ifindex);
3306 3307
		return TC_ACT_SHOT;
	}
L
Linus Torvalds 已提交
3308

3309 3310
	skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
	skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
L
Linus Torvalds 已提交
3311

3312
	q = rxq->qdisc;
3313
	if (q != &noop_qdisc) {
3314
		spin_lock(qdisc_lock(q));
3315 3316
		if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
			result = qdisc_enqueue_root(skb, q);
3317 3318
		spin_unlock(qdisc_lock(q));
	}
3319 3320 3321

	return result;
}
3322

3323 3324 3325 3326
static inline struct sk_buff *handle_ing(struct sk_buff *skb,
					 struct packet_type **pt_prev,
					 int *ret, struct net_device *orig_dev)
{
3327 3328 3329
	struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);

	if (!rxq || rxq->qdisc == &noop_qdisc)
3330
		goto out;
L
Linus Torvalds 已提交
3331

3332 3333 3334
	if (*pt_prev) {
		*ret = deliver_skb(skb, *pt_prev, orig_dev);
		*pt_prev = NULL;
L
Linus Torvalds 已提交
3335 3336
	}

3337
	switch (ing_filter(skb, rxq)) {
3338 3339 3340 3341 3342 3343 3344 3345 3346
	case TC_ACT_SHOT:
	case TC_ACT_STOLEN:
		kfree_skb(skb);
		return NULL;
	}

out:
	skb->tc_verd = 0;
	return skb;
L
Linus Torvalds 已提交
3347 3348 3349
}
#endif

3350 3351 3352 3353
/**
 *	netdev_rx_handler_register - register receive handler
 *	@dev: device to register a handler for
 *	@rx_handler: receive handler to register
J
Jiri Pirko 已提交
3354
 *	@rx_handler_data: data pointer that is used by rx handler
3355 3356 3357 3358 3359 3360
 *
 *	Register a receive hander for a device. This handler will then be
 *	called from __netif_receive_skb. A negative errno code is returned
 *	on a failure.
 *
 *	The caller must hold the rtnl_mutex.
3361 3362
 *
 *	For a general description of rx_handler, see enum rx_handler_result.
3363 3364
 */
int netdev_rx_handler_register(struct net_device *dev,
J
Jiri Pirko 已提交
3365 3366
			       rx_handler_func_t *rx_handler,
			       void *rx_handler_data)
3367 3368 3369 3370 3371 3372
{
	ASSERT_RTNL();

	if (dev->rx_handler)
		return -EBUSY;

3373
	/* Note: rx_handler_data must be set before rx_handler */
J
Jiri Pirko 已提交
3374
	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3375 3376 3377 3378 3379 3380 3381 3382 3383 3384
	rcu_assign_pointer(dev->rx_handler, rx_handler);

	return 0;
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_register);

/**
 *	netdev_rx_handler_unregister - unregister receive handler
 *	@dev: device to unregister a handler from
 *
K
Kusanagi Kouichi 已提交
3385
 *	Unregister a receive handler from a device.
3386 3387 3388 3389 3390 3391 3392
 *
 *	The caller must hold the rtnl_mutex.
 */
void netdev_rx_handler_unregister(struct net_device *dev)
{

	ASSERT_RTNL();
3393
	RCU_INIT_POINTER(dev->rx_handler, NULL);
3394 3395 3396 3397 3398
	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
	 * section has a guarantee to see a non NULL rx_handler_data
	 * as well.
	 */
	synchronize_net();
3399
	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3400 3401 3402
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);

3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413
/*
 * Limit the use of PFMEMALLOC reserves to those protocols that implement
 * the special handling of PFMEMALLOC skbs.
 */
static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
{
	switch (skb->protocol) {
	case __constant_htons(ETH_P_ARP):
	case __constant_htons(ETH_P_IP):
	case __constant_htons(ETH_P_IPV6):
	case __constant_htons(ETH_P_8021Q):
P
Patrick McHardy 已提交
3414
	case __constant_htons(ETH_P_8021AD):
3415 3416 3417 3418 3419 3420
		return true;
	default:
		return false;
	}
}

3421
static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
L
Linus Torvalds 已提交
3422 3423
{
	struct packet_type *ptype, *pt_prev;
3424
	rx_handler_func_t *rx_handler;
D
David S. Miller 已提交
3425
	struct net_device *orig_dev;
3426
	struct net_device *null_or_dev;
3427
	bool deliver_exact = false;
L
Linus Torvalds 已提交
3428
	int ret = NET_RX_DROP;
A
Al Viro 已提交
3429
	__be16 type;
L
Linus Torvalds 已提交
3430

3431
	net_timestamp_check(!netdev_tstamp_prequeue, skb);
3432

3433
	trace_netif_receive_skb(skb);
3434

L
Linus Torvalds 已提交
3435
	/* if we've gotten here through NAPI, check netpoll */
3436
	if (netpoll_receive_skb(skb))
3437
		goto out;
L
Linus Torvalds 已提交
3438

J
Joe Eykholt 已提交
3439
	orig_dev = skb->dev;
3440

3441
	skb_reset_network_header(skb);
3442 3443
	if (!skb_transport_header_was_set(skb))
		skb_reset_transport_header(skb);
3444
	skb_reset_mac_len(skb);
L
Linus Torvalds 已提交
3445 3446 3447 3448 3449

	pt_prev = NULL;

	rcu_read_lock();

3450
another_round:
3451
	skb->skb_iif = skb->dev->ifindex;
3452 3453 3454

	__this_cpu_inc(softnet_data.processed);

P
Patrick McHardy 已提交
3455 3456
	if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
	    skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
3457 3458
		skb = vlan_untag(skb);
		if (unlikely(!skb))
3459
			goto unlock;
3460 3461
	}

L
Linus Torvalds 已提交
3462 3463 3464 3465 3466 3467 3468
#ifdef CONFIG_NET_CLS_ACT
	if (skb->tc_verd & TC_NCLS) {
		skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
		goto ncls;
	}
#endif

3469
	if (pfmemalloc)
3470 3471
		goto skip_taps;

L
Linus Torvalds 已提交
3472
	list_for_each_entry_rcu(ptype, &ptype_all, list) {
3473
		if (!ptype->dev || ptype->dev == skb->dev) {
3474
			if (pt_prev)
D
David S. Miller 已提交
3475
				ret = deliver_skb(skb, pt_prev, orig_dev);
L
Linus Torvalds 已提交
3476 3477 3478 3479
			pt_prev = ptype;
		}
	}

3480
skip_taps:
L
Linus Torvalds 已提交
3481
#ifdef CONFIG_NET_CLS_ACT
3482 3483
	skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
	if (!skb)
3484
		goto unlock;
L
Linus Torvalds 已提交
3485 3486 3487
ncls:
#endif

3488
	if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
3489 3490
		goto drop;

3491 3492 3493 3494 3495
	if (vlan_tx_tag_present(skb)) {
		if (pt_prev) {
			ret = deliver_skb(skb, pt_prev, orig_dev);
			pt_prev = NULL;
		}
3496
		if (vlan_do_receive(&skb))
3497 3498
			goto another_round;
		else if (unlikely(!skb))
3499
			goto unlock;
3500 3501
	}

3502
	rx_handler = rcu_dereference(skb->dev->rx_handler);
3503 3504 3505 3506 3507
	if (rx_handler) {
		if (pt_prev) {
			ret = deliver_skb(skb, pt_prev, orig_dev);
			pt_prev = NULL;
		}
3508 3509
		switch (rx_handler(&skb)) {
		case RX_HANDLER_CONSUMED:
3510
			ret = NET_RX_SUCCESS;
3511
			goto unlock;
3512
		case RX_HANDLER_ANOTHER:
3513
			goto another_round;
3514 3515 3516 3517 3518 3519 3520
		case RX_HANDLER_EXACT:
			deliver_exact = true;
		case RX_HANDLER_PASS:
			break;
		default:
			BUG();
		}
3521
	}
L
Linus Torvalds 已提交
3522

3523 3524 3525
	if (vlan_tx_nonzero_tag_present(skb))
		skb->pkt_type = PACKET_OTHERHOST;

3526
	/* deliver only exact match when indicated */
3527
	null_or_dev = deliver_exact ? skb->dev : NULL;
3528

L
Linus Torvalds 已提交
3529
	type = skb->protocol;
3530 3531
	list_for_each_entry_rcu(ptype,
			&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3532
		if (ptype->type == type &&
3533 3534
		    (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
		     ptype->dev == orig_dev)) {
3535
			if (pt_prev)
D
David S. Miller 已提交
3536
				ret = deliver_skb(skb, pt_prev, orig_dev);
L
Linus Torvalds 已提交
3537 3538 3539 3540 3541
			pt_prev = ptype;
		}
	}

	if (pt_prev) {
3542
		if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
3543
			goto drop;
3544 3545
		else
			ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
L
Linus Torvalds 已提交
3546
	} else {
3547
drop:
3548
		atomic_long_inc(&skb->dev->rx_dropped);
L
Linus Torvalds 已提交
3549 3550 3551 3552 3553 3554 3555
		kfree_skb(skb);
		/* Jamal, now you will not able to escape explaining
		 * me how you were going to use this. :-)
		 */
		ret = NET_RX_DROP;
	}

3556
unlock:
L
Linus Torvalds 已提交
3557
	rcu_read_unlock();
3558
out:
3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583
	return ret;
}

static int __netif_receive_skb(struct sk_buff *skb)
{
	int ret;

	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
		unsigned long pflags = current->flags;

		/*
		 * PFMEMALLOC skbs are special, they should
		 * - be delivered to SOCK_MEMALLOC sockets only
		 * - stay away from userspace
		 * - have bounded memory usage
		 *
		 * Use PF_MEMALLOC as this saves us from propagating the allocation
		 * context down to all allocation sites.
		 */
		current->flags |= PF_MEMALLOC;
		ret = __netif_receive_skb_core(skb, true);
		tsk_restore_flags(current, pflags, PF_MEMALLOC);
	} else
		ret = __netif_receive_skb_core(skb, false);

L
Linus Torvalds 已提交
3584 3585
	return ret;
}
T
Tom Herbert 已提交
3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603

/**
 *	netif_receive_skb - process receive buffer from network
 *	@skb: buffer to process
 *
 *	netif_receive_skb() is the main receive data processing function.
 *	It always succeeds. The buffer may be dropped during processing
 *	for congestion control or by the protocol layers.
 *
 *	This function may only be called from softirq context and interrupts
 *	should be enabled.
 *
 *	Return values (usually ignored):
 *	NET_RX_SUCCESS: no congestion
 *	NET_RX_DROP: packet was dropped
 */
int netif_receive_skb(struct sk_buff *skb)
{
3604
	net_timestamp_check(netdev_tstamp_prequeue, skb);
E
Eric Dumazet 已提交
3605

3606 3607 3608
	if (skb_defer_rx_timestamp(skb))
		return NET_RX_SUCCESS;

E
Eric Dumazet 已提交
3609
#ifdef CONFIG_RPS
3610
	if (static_key_false(&rps_needed)) {
E
Eric Dumazet 已提交
3611 3612
		struct rps_dev_flow voidflow, *rflow = &voidflow;
		int cpu, ret;
T
Tom Herbert 已提交
3613

E
Eric Dumazet 已提交
3614 3615 3616
		rcu_read_lock();

		cpu = get_rps_cpu(skb->dev, skb, &rflow);
T
Tom Herbert 已提交
3617

E
Eric Dumazet 已提交
3618 3619 3620
		if (cpu >= 0) {
			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
			rcu_read_unlock();
3621
			return ret;
E
Eric Dumazet 已提交
3622
		}
3623
		rcu_read_unlock();
T
Tom Herbert 已提交
3624
	}
3625
#endif
3626
	return __netif_receive_skb(skb);
T
Tom Herbert 已提交
3627
}
E
Eric Dumazet 已提交
3628
EXPORT_SYMBOL(netif_receive_skb);
L
Linus Torvalds 已提交
3629

E
Eric Dumazet 已提交
3630 3631 3632
/* Network device is going away, flush any packets still pending
 * Called with irqs disabled.
 */
3633
static void flush_backlog(void *arg)
3634
{
3635
	struct net_device *dev = arg;
E
Eric Dumazet 已提交
3636
	struct softnet_data *sd = &__get_cpu_var(softnet_data);
3637 3638
	struct sk_buff *skb, *tmp;

E
Eric Dumazet 已提交
3639
	rps_lock(sd);
3640
	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3641
		if (skb->dev == dev) {
E
Eric Dumazet 已提交
3642
			__skb_unlink(skb, &sd->input_pkt_queue);
3643
			kfree_skb(skb);
3644
			input_queue_head_incr(sd);
3645
		}
3646
	}
E
Eric Dumazet 已提交
3647
	rps_unlock(sd);
3648 3649 3650 3651 3652

	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
		if (skb->dev == dev) {
			__skb_unlink(skb, &sd->process_queue);
			kfree_skb(skb);
3653
			input_queue_head_incr(sd);
3654 3655
		}
	}
3656 3657
}

3658 3659
static int napi_gro_complete(struct sk_buff *skb)
{
3660
	struct packet_offload *ptype;
3661
	__be16 type = skb->protocol;
3662
	struct list_head *head = &offload_base;
3663 3664
	int err = -ENOENT;

3665 3666
	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));

3667 3668
	if (NAPI_GRO_CB(skb)->count == 1) {
		skb_shinfo(skb)->gso_size = 0;
3669
		goto out;
3670
	}
3671 3672 3673

	rcu_read_lock();
	list_for_each_entry_rcu(ptype, head, list) {
3674
		if (ptype->type != type || !ptype->callbacks.gro_complete)
3675 3676
			continue;

3677
		err = ptype->callbacks.gro_complete(skb);
3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691
		break;
	}
	rcu_read_unlock();

	if (err) {
		WARN_ON(&ptype->list == head);
		kfree_skb(skb);
		return NET_RX_SUCCESS;
	}

out:
	return netif_receive_skb(skb);
}

3692 3693 3694 3695 3696
/* napi->gro_list contains packets ordered by age.
 * youngest packets at the head of it.
 * Complete skbs in reverse order to reduce latencies.
 */
void napi_gro_flush(struct napi_struct *napi, bool flush_old)
3697
{
3698
	struct sk_buff *skb, *prev = NULL;
3699

3700 3701 3702 3703 3704 3705 3706
	/* scan list and build reverse chain */
	for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
		skb->prev = prev;
		prev = skb;
	}

	for (skb = prev; skb; skb = prev) {
3707
		skb->next = NULL;
3708 3709 3710 3711 3712

		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
			return;

		prev = skb->prev;
3713
		napi_gro_complete(skb);
3714
		napi->gro_count--;
3715 3716 3717 3718
	}

	napi->gro_list = NULL;
}
E
Eric Dumazet 已提交
3719
EXPORT_SYMBOL(napi_gro_flush);
3720

E
Eric Dumazet 已提交
3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742
static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
{
	struct sk_buff *p;
	unsigned int maclen = skb->dev->hard_header_len;

	for (p = napi->gro_list; p; p = p->next) {
		unsigned long diffs;

		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
		diffs |= p->vlan_tci ^ skb->vlan_tci;
		if (maclen == ETH_HLEN)
			diffs |= compare_ether_header(skb_mac_header(p),
						      skb_gro_mac_header(skb));
		else if (!diffs)
			diffs = memcmp(skb_mac_header(p),
				       skb_gro_mac_header(skb),
				       maclen);
		NAPI_GRO_CB(p)->same_flow = !diffs;
		NAPI_GRO_CB(p)->flush = 0;
	}
}

R
Rami Rosen 已提交
3743
static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3744 3745
{
	struct sk_buff **pp = NULL;
3746
	struct packet_offload *ptype;
3747
	__be16 type = skb->protocol;
3748
	struct list_head *head = &offload_base;
H
Herbert Xu 已提交
3749
	int same_flow;
3750
	enum gro_result ret;
3751

3752
	if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3753 3754
		goto normal;

3755
	if (skb_is_gso(skb) || skb_has_frag_list(skb))
3756 3757
		goto normal;

E
Eric Dumazet 已提交
3758 3759
	gro_list_prepare(napi, skb);

3760 3761
	rcu_read_lock();
	list_for_each_entry_rcu(ptype, head, list) {
3762
		if (ptype->type != type || !ptype->callbacks.gro_receive)
3763 3764
			continue;

3765
		skb_set_network_header(skb, skb_gro_offset(skb));
3766
		skb_reset_mac_len(skb);
3767 3768
		NAPI_GRO_CB(skb)->same_flow = 0;
		NAPI_GRO_CB(skb)->flush = 0;
H
Herbert Xu 已提交
3769
		NAPI_GRO_CB(skb)->free = 0;
3770

3771
		pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
3772 3773 3774 3775 3776 3777 3778
		break;
	}
	rcu_read_unlock();

	if (&ptype->list == head)
		goto normal;

H
Herbert Xu 已提交
3779
	same_flow = NAPI_GRO_CB(skb)->same_flow;
3780
	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
H
Herbert Xu 已提交
3781

3782 3783 3784 3785 3786 3787
	if (pp) {
		struct sk_buff *nskb = *pp;

		*pp = nskb->next;
		nskb->next = NULL;
		napi_gro_complete(nskb);
3788
		napi->gro_count--;
3789 3790
	}

H
Herbert Xu 已提交
3791
	if (same_flow)
3792 3793
		goto ok;

3794
	if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3795 3796
		goto normal;

3797
	napi->gro_count++;
3798
	NAPI_GRO_CB(skb)->count = 1;
3799
	NAPI_GRO_CB(skb)->age = jiffies;
3800
	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3801 3802
	skb->next = napi->gro_list;
	napi->gro_list = skb;
3803
	ret = GRO_HELD;
3804

3805
pull:
H
Herbert Xu 已提交
3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816
	if (skb_headlen(skb) < skb_gro_offset(skb)) {
		int grow = skb_gro_offset(skb) - skb_headlen(skb);

		BUG_ON(skb->end - skb->tail < grow);

		memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);

		skb->tail += grow;
		skb->data_len -= grow;

		skb_shinfo(skb)->frags[0].page_offset += grow;
E
Eric Dumazet 已提交
3817
		skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
H
Herbert Xu 已提交
3818

E
Eric Dumazet 已提交
3819
		if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
3820
			skb_frag_unref(skb, 0);
H
Herbert Xu 已提交
3821 3822
			memmove(skb_shinfo(skb)->frags,
				skb_shinfo(skb)->frags + 1,
3823
				--skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
H
Herbert Xu 已提交
3824
		}
3825 3826
	}

3827
ok:
3828
	return ret;
3829 3830

normal:
3831 3832
	ret = GRO_NORMAL;
	goto pull;
H
Herbert Xu 已提交
3833
}
3834

H
Herbert Xu 已提交
3835

R
Rami Rosen 已提交
3836
static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
H
Herbert Xu 已提交
3837
{
3838 3839
	switch (ret) {
	case GRO_NORMAL:
3840 3841 3842
		if (netif_receive_skb(skb))
			ret = GRO_DROP;
		break;
H
Herbert Xu 已提交
3843

3844
	case GRO_DROP:
H
Herbert Xu 已提交
3845 3846
		kfree_skb(skb);
		break;
3847

3848
	case GRO_MERGED_FREE:
3849 3850 3851 3852
		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
			kmem_cache_free(skbuff_head_cache, skb);
		else
			__kfree_skb(skb);
3853 3854
		break;

3855 3856 3857
	case GRO_HELD:
	case GRO_MERGED:
		break;
H
Herbert Xu 已提交
3858 3859
	}

3860
	return ret;
3861 3862
}

3863
static void skb_gro_reset_offset(struct sk_buff *skb)
3864
{
3865 3866 3867
	const struct skb_shared_info *pinfo = skb_shinfo(skb);
	const skb_frag_t *frag0 = &pinfo->frags[0];

3868 3869
	NAPI_GRO_CB(skb)->data_offset = 0;
	NAPI_GRO_CB(skb)->frag0 = NULL;
3870
	NAPI_GRO_CB(skb)->frag0_len = 0;
3871

3872
	if (skb->mac_header == skb->tail &&
3873 3874 3875 3876
	    pinfo->nr_frags &&
	    !PageHighMem(skb_frag_page(frag0))) {
		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
		NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
3877
	}
3878 3879
}

3880
gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3881
{
3882 3883
	skb_gro_reset_offset(skb);

E
Eric Dumazet 已提交
3884
	return napi_skb_finish(dev_gro_receive(napi, skb), skb);
3885 3886 3887
}
EXPORT_SYMBOL(napi_gro_receive);

S
stephen hemminger 已提交
3888
static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3889 3890
{
	__skb_pull(skb, skb_headlen(skb));
3891 3892
	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
3893
	skb->vlan_tci = 0;
H
Herbert Xu 已提交
3894
	skb->dev = napi->dev;
A
Andy Gospodarek 已提交
3895
	skb->skb_iif = 0;
3896 3897 3898 3899

	napi->skb = skb;
}

3900
struct sk_buff *napi_get_frags(struct napi_struct *napi)
H
Herbert Xu 已提交
3901 3902 3903 3904
{
	struct sk_buff *skb = napi->skb;

	if (!skb) {
3905 3906 3907
		skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
		if (skb)
			napi->skb = skb;
3908
	}
3909 3910
	return skb;
}
3911
EXPORT_SYMBOL(napi_get_frags);
3912

R
Rami Rosen 已提交
3913
static gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3914
			       gro_result_t ret)
3915
{
3916 3917
	switch (ret) {
	case GRO_NORMAL:
3918
	case GRO_HELD:
3919
		skb->protocol = eth_type_trans(skb, skb->dev);
3920

3921 3922 3923 3924
		if (ret == GRO_HELD)
			skb_gro_pull(skb, -ETH_HLEN);
		else if (netif_receive_skb(skb))
			ret = GRO_DROP;
3925
		break;
H
Herbert Xu 已提交
3926

3927 3928 3929 3930
	case GRO_DROP:
	case GRO_MERGED_FREE:
		napi_reuse_skb(napi, skb);
		break;
3931 3932 3933

	case GRO_MERGED:
		break;
3934
	}
H
Herbert Xu 已提交
3935

3936
	return ret;
H
Herbert Xu 已提交
3937
}
3938

E
Eric Dumazet 已提交
3939
static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3940 3941 3942
{
	struct sk_buff *skb = napi->skb;
	struct ethhdr *eth;
3943 3944
	unsigned int hlen;
	unsigned int off;
3945 3946 3947 3948 3949 3950

	napi->skb = NULL;

	skb_reset_mac_header(skb);
	skb_gro_reset_offset(skb);

3951 3952 3953 3954 3955 3956 3957 3958 3959 3960
	off = skb_gro_offset(skb);
	hlen = off + sizeof(*eth);
	eth = skb_gro_header_fast(skb, off);
	if (skb_gro_header_hard(skb, hlen)) {
		eth = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!eth)) {
			napi_reuse_skb(napi, skb);
			skb = NULL;
			goto out;
		}
3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974
	}

	skb_gro_pull(skb, sizeof(*eth));

	/*
	 * This works because the only protocols we care about don't require
	 * special handling.  We'll fix it up properly at the end.
	 */
	skb->protocol = eth->h_proto;

out:
	return skb;
}

3975
gro_result_t napi_gro_frags(struct napi_struct *napi)
3976
{
3977
	struct sk_buff *skb = napi_frags_skb(napi);
3978 3979

	if (!skb)
3980
		return GRO_DROP;
3981

E
Eric Dumazet 已提交
3982
	return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
3983
}
H
Herbert Xu 已提交
3984 3985
EXPORT_SYMBOL(napi_gro_frags);

3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013
/*
 * net_rps_action sends any pending IPI's for rps.
 * Note: called with local irq disabled, but exits with local irq enabled.
 */
static void net_rps_action_and_irq_enable(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
	struct softnet_data *remsd = sd->rps_ipi_list;

	if (remsd) {
		sd->rps_ipi_list = NULL;

		local_irq_enable();

		/* Send pending IPI's to kick RPS processing on remote cpus. */
		while (remsd) {
			struct softnet_data *next = remsd->rps_ipi_next;

			if (cpu_online(remsd->cpu))
				__smp_call_function_single(remsd->cpu,
							   &remsd->csd, 0);
			remsd = next;
		}
	} else
#endif
		local_irq_enable();
}

4014
static int process_backlog(struct napi_struct *napi, int quota)
L
Linus Torvalds 已提交
4015 4016
{
	int work = 0;
E
Eric Dumazet 已提交
4017
	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
L
Linus Torvalds 已提交
4018

4019 4020 4021 4022 4023 4024 4025 4026 4027
#ifdef CONFIG_RPS
	/* Check if we have pending ipi, its better to send them now,
	 * not waiting net_rx_action() end.
	 */
	if (sd->rps_ipi_list) {
		local_irq_disable();
		net_rps_action_and_irq_enable(sd);
	}
#endif
4028
	napi->weight = weight_p;
4029 4030
	local_irq_disable();
	while (work < quota) {
L
Linus Torvalds 已提交
4031
		struct sk_buff *skb;
4032 4033 4034 4035 4036 4037
		unsigned int qlen;

		while ((skb = __skb_dequeue(&sd->process_queue))) {
			local_irq_enable();
			__netif_receive_skb(skb);
			local_irq_disable();
4038 4039 4040 4041 4042
			input_queue_head_incr(sd);
			if (++work >= quota) {
				local_irq_enable();
				return work;
			}
4043
		}
L
Linus Torvalds 已提交
4044

E
Eric Dumazet 已提交
4045
		rps_lock(sd);
4046
		qlen = skb_queue_len(&sd->input_pkt_queue);
4047
		if (qlen)
4048 4049
			skb_queue_splice_tail_init(&sd->input_pkt_queue,
						   &sd->process_queue);
4050

4051
		if (qlen < quota - work) {
E
Eric Dumazet 已提交
4052 4053 4054 4055 4056 4057 4058 4059 4060 4061
			/*
			 * Inline a custom version of __napi_complete().
			 * only current cpu owns and manipulates this napi,
			 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
			 * we can use a plain write instead of clear_bit(),
			 * and we dont need an smp_mb() memory barrier.
			 */
			list_del(&napi->poll_list);
			napi->state = 0;

4062
			quota = work + qlen;
4063
		}
E
Eric Dumazet 已提交
4064
		rps_unlock(sd);
4065 4066
	}
	local_irq_enable();
L
Linus Torvalds 已提交
4067

4068 4069
	return work;
}
L
Linus Torvalds 已提交
4070

4071 4072
/**
 * __napi_schedule - schedule for receive
4073
 * @n: entry to schedule
4074 4075 4076
 *
 * The entry's receive function will be scheduled to run
 */
H
Harvey Harrison 已提交
4077
void __napi_schedule(struct napi_struct *n)
4078 4079
{
	unsigned long flags;
L
Linus Torvalds 已提交
4080

4081
	local_irq_save(flags);
E
Eric Dumazet 已提交
4082
	____napi_schedule(&__get_cpu_var(softnet_data), n);
4083
	local_irq_restore(flags);
L
Linus Torvalds 已提交
4084
}
4085 4086
EXPORT_SYMBOL(__napi_schedule);

4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108
void __napi_complete(struct napi_struct *n)
{
	BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
	BUG_ON(n->gro_list);

	list_del(&n->poll_list);
	smp_mb__before_clear_bit();
	clear_bit(NAPI_STATE_SCHED, &n->state);
}
EXPORT_SYMBOL(__napi_complete);

void napi_complete(struct napi_struct *n)
{
	unsigned long flags;

	/*
	 * don't let napi dequeue from the cpu poll list
	 * just in case its running on a different cpu
	 */
	if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
		return;

4109
	napi_gro_flush(n, false);
4110 4111 4112 4113 4114 4115 4116 4117 4118 4119
	local_irq_save(flags);
	__napi_complete(n);
	local_irq_restore(flags);
}
EXPORT_SYMBOL(napi_complete);

void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
		    int (*poll)(struct napi_struct *, int), int weight)
{
	INIT_LIST_HEAD(&napi->poll_list);
4120
	napi->gro_count = 0;
4121
	napi->gro_list = NULL;
H
Herbert Xu 已提交
4122
	napi->skb = NULL;
4123
	napi->poll = poll;
E
Eric Dumazet 已提交
4124 4125 4126
	if (weight > NAPI_POLL_WEIGHT)
		pr_err_once("netif_napi_add() called with weight %d on device %s\n",
			    weight, dev->name);
4127 4128 4129
	napi->weight = weight;
	list_add(&napi->dev_list, &dev->napi_list);
	napi->dev = dev;
H
Herbert Xu 已提交
4130
#ifdef CONFIG_NETPOLL
4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141
	spin_lock_init(&napi->poll_lock);
	napi->poll_owner = -1;
#endif
	set_bit(NAPI_STATE_SCHED, &napi->state);
}
EXPORT_SYMBOL(netif_napi_add);

void netif_napi_del(struct napi_struct *napi)
{
	struct sk_buff *skb, *next;

4142
	list_del_init(&napi->dev_list);
4143
	napi_free_frags(napi);
4144 4145 4146 4147 4148 4149 4150 4151

	for (skb = napi->gro_list; skb; skb = next) {
		next = skb->next;
		skb->next = NULL;
		kfree_skb(skb);
	}

	napi->gro_list = NULL;
4152
	napi->gro_count = 0;
4153 4154 4155
}
EXPORT_SYMBOL(netif_napi_del);

L
Linus Torvalds 已提交
4156 4157
static void net_rx_action(struct softirq_action *h)
{
4158
	struct softnet_data *sd = &__get_cpu_var(softnet_data);
4159
	unsigned long time_limit = jiffies + 2;
4160
	int budget = netdev_budget;
4161 4162
	void *have;

L
Linus Torvalds 已提交
4163 4164
	local_irq_disable();

4165
	while (!list_empty(&sd->poll_list)) {
4166 4167
		struct napi_struct *n;
		int work, weight;
L
Linus Torvalds 已提交
4168

4169
		/* If softirq window is exhuasted then punt.
4170 4171
		 * Allow this to run for 2 jiffies since which will allow
		 * an average latency of 1.5/HZ.
4172
		 */
4173
		if (unlikely(budget <= 0 || time_after_eq(jiffies, time_limit)))
L
Linus Torvalds 已提交
4174 4175 4176 4177
			goto softnet_break;

		local_irq_enable();

4178 4179 4180 4181 4182
		/* Even though interrupts have been re-enabled, this
		 * access is safe because interrupts can only add new
		 * entries to the tail of this list, and only ->poll()
		 * calls can remove this head entry from the list.
		 */
4183
		n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
L
Linus Torvalds 已提交
4184

4185 4186 4187 4188
		have = netpoll_poll_lock(n);

		weight = n->weight;

4189 4190 4191 4192
		/* This NAPI_STATE_SCHED test is for avoiding a race
		 * with netpoll's poll_napi().  Only the entity which
		 * obtains the lock and sees NAPI_STATE_SCHED set will
		 * actually make the ->poll() call.  Therefore we avoid
L
Lucas De Marchi 已提交
4193
		 * accidentally calling ->poll() when NAPI is not scheduled.
4194 4195
		 */
		work = 0;
4196
		if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4197
			work = n->poll(n, weight);
4198 4199
			trace_napi_poll(n);
		}
4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211

		WARN_ON_ONCE(work > weight);

		budget -= work;

		local_irq_disable();

		/* Drivers must not modify the NAPI state if they
		 * consume the entire weight.  In such cases this code
		 * still "owns" the NAPI instance and therefore can
		 * move the instance around on the list at-will.
		 */
4212
		if (unlikely(work == weight)) {
4213 4214 4215 4216
			if (unlikely(napi_disable_pending(n))) {
				local_irq_enable();
				napi_complete(n);
				local_irq_disable();
4217 4218 4219 4220 4221 4222 4223 4224 4225
			} else {
				if (n->gro_list) {
					/* flush too old packets
					 * If HZ < 1000, flush all packets.
					 */
					local_irq_enable();
					napi_gro_flush(n, HZ >= 1000);
					local_irq_disable();
				}
4226
				list_move_tail(&n->poll_list, &sd->poll_list);
4227
			}
4228
		}
4229 4230

		netpoll_poll_unlock(have);
L
Linus Torvalds 已提交
4231 4232
	}
out:
4233
	net_rps_action_and_irq_enable(sd);
T
Tom Herbert 已提交
4234

4235 4236 4237 4238 4239
#ifdef CONFIG_NET_DMA
	/*
	 * There may not be any more sk_buffs coming right now, so push
	 * any pending DMA copies to hardware
	 */
4240
	dma_issue_pending_all();
4241
#endif
4242

L
Linus Torvalds 已提交
4243 4244 4245
	return;

softnet_break:
C
Changli Gao 已提交
4246
	sd->time_squeeze++;
L
Linus Torvalds 已提交
4247 4248 4249 4250
	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
	goto out;
}

J
Jiri Pirko 已提交
4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413
struct netdev_upper {
	struct net_device *dev;
	bool master;
	struct list_head list;
	struct rcu_head rcu;
	struct list_head search_list;
};

static void __append_search_uppers(struct list_head *search_list,
				   struct net_device *dev)
{
	struct netdev_upper *upper;

	list_for_each_entry(upper, &dev->upper_dev_list, list) {
		/* check if this upper is not already in search list */
		if (list_empty(&upper->search_list))
			list_add_tail(&upper->search_list, search_list);
	}
}

static bool __netdev_search_upper_dev(struct net_device *dev,
				      struct net_device *upper_dev)
{
	LIST_HEAD(search_list);
	struct netdev_upper *upper;
	struct netdev_upper *tmp;
	bool ret = false;

	__append_search_uppers(&search_list, dev);
	list_for_each_entry(upper, &search_list, search_list) {
		if (upper->dev == upper_dev) {
			ret = true;
			break;
		}
		__append_search_uppers(&search_list, upper->dev);
	}
	list_for_each_entry_safe(upper, tmp, &search_list, search_list)
		INIT_LIST_HEAD(&upper->search_list);
	return ret;
}

static struct netdev_upper *__netdev_find_upper(struct net_device *dev,
						struct net_device *upper_dev)
{
	struct netdev_upper *upper;

	list_for_each_entry(upper, &dev->upper_dev_list, list) {
		if (upper->dev == upper_dev)
			return upper;
	}
	return NULL;
}

/**
 * netdev_has_upper_dev - Check if device is linked to an upper device
 * @dev: device
 * @upper_dev: upper device to check
 *
 * Find out if a device is linked to specified upper device and return true
 * in case it is. Note that this checks only immediate upper device,
 * not through a complete stack of devices. The caller must hold the RTNL lock.
 */
bool netdev_has_upper_dev(struct net_device *dev,
			  struct net_device *upper_dev)
{
	ASSERT_RTNL();

	return __netdev_find_upper(dev, upper_dev);
}
EXPORT_SYMBOL(netdev_has_upper_dev);

/**
 * netdev_has_any_upper_dev - Check if device is linked to some device
 * @dev: device
 *
 * Find out if a device is linked to an upper device and return true in case
 * it is. The caller must hold the RTNL lock.
 */
bool netdev_has_any_upper_dev(struct net_device *dev)
{
	ASSERT_RTNL();

	return !list_empty(&dev->upper_dev_list);
}
EXPORT_SYMBOL(netdev_has_any_upper_dev);

/**
 * netdev_master_upper_dev_get - Get master upper device
 * @dev: device
 *
 * Find a master upper device and return pointer to it or NULL in case
 * it's not there. The caller must hold the RTNL lock.
 */
struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
{
	struct netdev_upper *upper;

	ASSERT_RTNL();

	if (list_empty(&dev->upper_dev_list))
		return NULL;

	upper = list_first_entry(&dev->upper_dev_list,
				 struct netdev_upper, list);
	if (likely(upper->master))
		return upper->dev;
	return NULL;
}
EXPORT_SYMBOL(netdev_master_upper_dev_get);

/**
 * netdev_master_upper_dev_get_rcu - Get master upper device
 * @dev: device
 *
 * Find a master upper device and return pointer to it or NULL in case
 * it's not there. The caller must hold the RCU read lock.
 */
struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
{
	struct netdev_upper *upper;

	upper = list_first_or_null_rcu(&dev->upper_dev_list,
				       struct netdev_upper, list);
	if (upper && likely(upper->master))
		return upper->dev;
	return NULL;
}
EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);

static int __netdev_upper_dev_link(struct net_device *dev,
				   struct net_device *upper_dev, bool master)
{
	struct netdev_upper *upper;

	ASSERT_RTNL();

	if (dev == upper_dev)
		return -EBUSY;

	/* To prevent loops, check if dev is not upper device to upper_dev. */
	if (__netdev_search_upper_dev(upper_dev, dev))
		return -EBUSY;

	if (__netdev_find_upper(dev, upper_dev))
		return -EEXIST;

	if (master && netdev_master_upper_dev_get(dev))
		return -EBUSY;

	upper = kmalloc(sizeof(*upper), GFP_KERNEL);
	if (!upper)
		return -ENOMEM;

	upper->dev = upper_dev;
	upper->master = master;
	INIT_LIST_HEAD(&upper->search_list);

	/* Ensure that master upper link is always the first item in list. */
	if (master)
		list_add_rcu(&upper->list, &dev->upper_dev_list);
	else
		list_add_tail_rcu(&upper->list, &dev->upper_dev_list);
	dev_hold(upper_dev);
4414
	call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
J
Jiri Pirko 已提交
4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473
	return 0;
}

/**
 * netdev_upper_dev_link - Add a link to the upper device
 * @dev: device
 * @upper_dev: new upper device
 *
 * Adds a link to device which is upper to this one. The caller must hold
 * the RTNL lock. On a failure a negative errno code is returned.
 * On success the reference counts are adjusted and the function
 * returns zero.
 */
int netdev_upper_dev_link(struct net_device *dev,
			  struct net_device *upper_dev)
{
	return __netdev_upper_dev_link(dev, upper_dev, false);
}
EXPORT_SYMBOL(netdev_upper_dev_link);

/**
 * netdev_master_upper_dev_link - Add a master link to the upper device
 * @dev: device
 * @upper_dev: new upper device
 *
 * Adds a link to device which is upper to this one. In this case, only
 * one master upper device can be linked, although other non-master devices
 * might be linked as well. The caller must hold the RTNL lock.
 * On a failure a negative errno code is returned. On success the reference
 * counts are adjusted and the function returns zero.
 */
int netdev_master_upper_dev_link(struct net_device *dev,
				 struct net_device *upper_dev)
{
	return __netdev_upper_dev_link(dev, upper_dev, true);
}
EXPORT_SYMBOL(netdev_master_upper_dev_link);

/**
 * netdev_upper_dev_unlink - Removes a link to upper device
 * @dev: device
 * @upper_dev: new upper device
 *
 * Removes a link to device which is upper to this one. The caller must hold
 * the RTNL lock.
 */
void netdev_upper_dev_unlink(struct net_device *dev,
			     struct net_device *upper_dev)
{
	struct netdev_upper *upper;

	ASSERT_RTNL();

	upper = __netdev_find_upper(dev, upper_dev);
	if (!upper)
		return;
	list_del_rcu(&upper->list);
	dev_put(upper_dev);
	kfree_rcu(upper, rcu);
4474
	call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
J
Jiri Pirko 已提交
4475 4476 4477
}
EXPORT_SYMBOL(netdev_upper_dev_unlink);

4478 4479
static void dev_change_rx_flags(struct net_device *dev, int flags)
{
4480 4481 4482 4483
	const struct net_device_ops *ops = dev->netdev_ops;

	if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
		ops->ndo_change_rx_flags(dev, flags);
4484 4485
}

4486
static int __dev_set_promiscuity(struct net_device *dev, int inc)
L
Linus Torvalds 已提交
4487
{
4488
	unsigned int old_flags = dev->flags;
4489 4490
	kuid_t uid;
	kgid_t gid;
L
Linus Torvalds 已提交
4491

4492 4493
	ASSERT_RTNL();

4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504
	dev->flags |= IFF_PROMISC;
	dev->promiscuity += inc;
	if (dev->promiscuity == 0) {
		/*
		 * Avoid overflow.
		 * If inc causes overflow, untouch promisc and return error.
		 */
		if (inc < 0)
			dev->flags &= ~IFF_PROMISC;
		else {
			dev->promiscuity -= inc;
4505 4506
			pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
				dev->name);
4507 4508 4509
			return -EOVERFLOW;
		}
	}
4510
	if (dev->flags != old_flags) {
4511 4512 4513
		pr_info("device %s %s promiscuous mode\n",
			dev->name,
			dev->flags & IFF_PROMISC ? "entered" : "left");
4514 4515
		if (audit_enabled) {
			current_uid_gid(&uid, &gid);
4516 4517 4518 4519 4520
			audit_log(current->audit_context, GFP_ATOMIC,
				AUDIT_ANOM_PROMISCUOUS,
				"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
				dev->name, (dev->flags & IFF_PROMISC),
				(old_flags & IFF_PROMISC),
4521
				from_kuid(&init_user_ns, audit_get_loginuid(current)),
4522 4523
				from_kuid(&init_user_ns, uid),
				from_kgid(&init_user_ns, gid),
4524
				audit_get_sessionid(current));
4525
		}
4526

4527
		dev_change_rx_flags(dev, IFF_PROMISC);
L
Linus Torvalds 已提交
4528
	}
4529
	return 0;
L
Linus Torvalds 已提交
4530 4531
}

4532 4533 4534 4535 4536 4537 4538 4539 4540
/**
 *	dev_set_promiscuity	- update promiscuity count on a device
 *	@dev: device
 *	@inc: modifier
 *
 *	Add or remove promiscuity from a device. While the count in the device
 *	remains above zero the interface remains promiscuous. Once it hits zero
 *	the device reverts back to normal filtering operation. A negative inc
 *	value is used to drop promiscuity on the device.
4541
 *	Return 0 if successful or a negative errno code on error.
4542
 */
4543
int dev_set_promiscuity(struct net_device *dev, int inc)
4544
{
4545
	unsigned int old_flags = dev->flags;
4546
	int err;
4547

4548
	err = __dev_set_promiscuity(dev, inc);
4549
	if (err < 0)
4550
		return err;
4551 4552
	if (dev->flags != old_flags)
		dev_set_rx_mode(dev);
4553
	return err;
4554
}
E
Eric Dumazet 已提交
4555
EXPORT_SYMBOL(dev_set_promiscuity);
4556

L
Linus Torvalds 已提交
4557 4558 4559 4560 4561 4562 4563 4564 4565 4566
/**
 *	dev_set_allmulti	- update allmulti count on a device
 *	@dev: device
 *	@inc: modifier
 *
 *	Add or remove reception of all multicast frames to a device. While the
 *	count in the device remains above zero the interface remains listening
 *	to all interfaces. Once it hits zero the device reverts back to normal
 *	filtering operation. A negative @inc value is used to drop the counter
 *	when releasing a resource needing all multicasts.
4567
 *	Return 0 if successful or a negative errno code on error.
L
Linus Torvalds 已提交
4568 4569
 */

4570
int dev_set_allmulti(struct net_device *dev, int inc)
L
Linus Torvalds 已提交
4571
{
4572
	unsigned int old_flags = dev->flags;
L
Linus Torvalds 已提交
4573

4574 4575
	ASSERT_RTNL();

L
Linus Torvalds 已提交
4576
	dev->flags |= IFF_ALLMULTI;
4577 4578 4579 4580 4581 4582 4583 4584 4585 4586
	dev->allmulti += inc;
	if (dev->allmulti == 0) {
		/*
		 * Avoid overflow.
		 * If inc causes overflow, untouch allmulti and return error.
		 */
		if (inc < 0)
			dev->flags &= ~IFF_ALLMULTI;
		else {
			dev->allmulti -= inc;
4587 4588
			pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
				dev->name);
4589 4590 4591
			return -EOVERFLOW;
		}
	}
4592
	if (dev->flags ^ old_flags) {
4593
		dev_change_rx_flags(dev, IFF_ALLMULTI);
4594
		dev_set_rx_mode(dev);
4595
	}
4596
	return 0;
4597
}
E
Eric Dumazet 已提交
4598
EXPORT_SYMBOL(dev_set_allmulti);
4599 4600 4601 4602

/*
 *	Upload unicast and multicast address lists to device and
 *	configure RX filtering. When the device doesn't support unicast
J
Joe Perches 已提交
4603
 *	filtering it is put in promiscuous mode while unicast addresses
4604 4605 4606 4607
 *	are present.
 */
void __dev_set_rx_mode(struct net_device *dev)
{
4608 4609
	const struct net_device_ops *ops = dev->netdev_ops;

4610 4611 4612 4613 4614
	/* dev_open will call this function so the list will stay sane. */
	if (!(dev->flags&IFF_UP))
		return;

	if (!netif_device_present(dev))
4615
		return;
4616

4617
	if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
4618 4619 4620
		/* Unicast addresses changes may only happen under the rtnl,
		 * therefore calling __dev_set_promiscuity here is safe.
		 */
4621
		if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4622
			__dev_set_promiscuity(dev, 1);
4623
			dev->uc_promisc = true;
4624
		} else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4625
			__dev_set_promiscuity(dev, -1);
4626
			dev->uc_promisc = false;
4627 4628
		}
	}
4629 4630 4631

	if (ops->ndo_set_rx_mode)
		ops->ndo_set_rx_mode(dev);
4632 4633 4634 4635
}

void dev_set_rx_mode(struct net_device *dev)
{
4636
	netif_addr_lock_bh(dev);
4637
	__dev_set_rx_mode(dev);
4638
	netif_addr_unlock_bh(dev);
L
Linus Torvalds 已提交
4639 4640
}

4641 4642 4643 4644 4645 4646
/**
 *	dev_get_flags - get flags reported to userspace
 *	@dev: device
 *
 *	Get the combination of flag bits exported through APIs to userspace.
 */
4647
unsigned int dev_get_flags(const struct net_device *dev)
L
Linus Torvalds 已提交
4648
{
4649
	unsigned int flags;
L
Linus Torvalds 已提交
4650 4651 4652

	flags = (dev->flags & ~(IFF_PROMISC |
				IFF_ALLMULTI |
S
Stefan Rompf 已提交
4653 4654 4655
				IFF_RUNNING |
				IFF_LOWER_UP |
				IFF_DORMANT)) |
L
Linus Torvalds 已提交
4656 4657 4658
		(dev->gflags & (IFF_PROMISC |
				IFF_ALLMULTI));

S
Stefan Rompf 已提交
4659 4660 4661 4662 4663 4664 4665 4666
	if (netif_running(dev)) {
		if (netif_oper_up(dev))
			flags |= IFF_RUNNING;
		if (netif_carrier_ok(dev))
			flags |= IFF_LOWER_UP;
		if (netif_dormant(dev))
			flags |= IFF_DORMANT;
	}
L
Linus Torvalds 已提交
4667 4668 4669

	return flags;
}
E
Eric Dumazet 已提交
4670
EXPORT_SYMBOL(dev_get_flags);
L
Linus Torvalds 已提交
4671

4672
int __dev_change_flags(struct net_device *dev, unsigned int flags)
L
Linus Torvalds 已提交
4673
{
4674
	unsigned int old_flags = dev->flags;
4675
	int ret;
L
Linus Torvalds 已提交
4676

4677 4678
	ASSERT_RTNL();

L
Linus Torvalds 已提交
4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692
	/*
	 *	Set the flags on our device.
	 */

	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
			       IFF_AUTOMEDIA)) |
		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
				    IFF_ALLMULTI));

	/*
	 *	Load in the correct multicast list now the flags have changed.
	 */

4693 4694
	if ((old_flags ^ flags) & IFF_MULTICAST)
		dev_change_rx_flags(dev, IFF_MULTICAST);
4695

4696
	dev_set_rx_mode(dev);
L
Linus Torvalds 已提交
4697 4698 4699 4700 4701 4702 4703 4704 4705

	/*
	 *	Have we downed the interface. We handle IFF_UP ourselves
	 *	according to user attempts to set it, rather than blindly
	 *	setting it.
	 */

	ret = 0;
	if ((old_flags ^ flags) & IFF_UP) {	/* Bit is different  ? */
4706
		ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
L
Linus Torvalds 已提交
4707 4708

		if (!ret)
4709
			dev_set_rx_mode(dev);
L
Linus Torvalds 已提交
4710 4711 4712
	}

	if ((flags ^ dev->gflags) & IFF_PROMISC) {
E
Eric Dumazet 已提交
4713 4714
		int inc = (flags & IFF_PROMISC) ? 1 : -1;

L
Linus Torvalds 已提交
4715 4716 4717 4718 4719 4720 4721 4722 4723
		dev->gflags ^= IFF_PROMISC;
		dev_set_promiscuity(dev, inc);
	}

	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
	   is important. Some (broken) drivers set IFF_PROMISC, when
	   IFF_ALLMULTI is requested not asking us and not reporting.
	 */
	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
E
Eric Dumazet 已提交
4724 4725
		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;

L
Linus Torvalds 已提交
4726 4727 4728 4729
		dev->gflags ^= IFF_ALLMULTI;
		dev_set_allmulti(dev, inc);
	}

4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756
	return ret;
}

void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
{
	unsigned int changes = dev->flags ^ old_flags;

	if (changes & IFF_UP) {
		if (dev->flags & IFF_UP)
			call_netdevice_notifiers(NETDEV_UP, dev);
		else
			call_netdevice_notifiers(NETDEV_DOWN, dev);
	}

	if (dev->flags & IFF_UP &&
	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
		call_netdevice_notifiers(NETDEV_CHANGE, dev);
}

/**
 *	dev_change_flags - change device settings
 *	@dev: device
 *	@flags: device state flags
 *
 *	Change settings on device based state flags. The flags are
 *	in the userspace exported format.
 */
4757
int dev_change_flags(struct net_device *dev, unsigned int flags)
4758
{
4759 4760
	int ret;
	unsigned int changes, old_flags = dev->flags;
4761 4762 4763 4764 4765 4766

	ret = __dev_change_flags(dev, flags);
	if (ret < 0)
		return ret;

	changes = old_flags ^ dev->flags;
4767 4768
	if (changes)
		rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
L
Linus Torvalds 已提交
4769

4770
	__dev_notify_flags(dev, old_flags);
L
Linus Torvalds 已提交
4771 4772
	return ret;
}
E
Eric Dumazet 已提交
4773
EXPORT_SYMBOL(dev_change_flags);
L
Linus Torvalds 已提交
4774

4775 4776 4777 4778 4779 4780 4781
/**
 *	dev_set_mtu - Change maximum transfer unit
 *	@dev: device
 *	@new_mtu: new transfer unit
 *
 *	Change the maximum transfer size of the network device.
 */
L
Linus Torvalds 已提交
4782 4783
int dev_set_mtu(struct net_device *dev, int new_mtu)
{
4784
	const struct net_device_ops *ops = dev->netdev_ops;
L
Linus Torvalds 已提交
4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797
	int err;

	if (new_mtu == dev->mtu)
		return 0;

	/*	MTU must be positive.	 */
	if (new_mtu < 0)
		return -EINVAL;

	if (!netif_device_present(dev))
		return -ENODEV;

	err = 0;
4798 4799
	if (ops->ndo_change_mtu)
		err = ops->ndo_change_mtu(dev, new_mtu);
L
Linus Torvalds 已提交
4800 4801
	else
		dev->mtu = new_mtu;
4802

4803
	if (!err)
4804
		call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
L
Linus Torvalds 已提交
4805 4806
	return err;
}
E
Eric Dumazet 已提交
4807
EXPORT_SYMBOL(dev_set_mtu);
L
Linus Torvalds 已提交
4808

4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819
/**
 *	dev_set_group - Change group this device belongs to
 *	@dev: device
 *	@new_group: group this device should belong to
 */
void dev_set_group(struct net_device *dev, int new_group)
{
	dev->group = new_group;
}
EXPORT_SYMBOL(dev_set_group);

4820 4821 4822 4823 4824 4825 4826
/**
 *	dev_set_mac_address - Change Media Access Control Address
 *	@dev: device
 *	@sa: new address
 *
 *	Change the hardware (MAC) address of the device
 */
L
Linus Torvalds 已提交
4827 4828
int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
{
4829
	const struct net_device_ops *ops = dev->netdev_ops;
L
Linus Torvalds 已提交
4830 4831
	int err;

4832
	if (!ops->ndo_set_mac_address)
L
Linus Torvalds 已提交
4833 4834 4835 4836 4837
		return -EOPNOTSUPP;
	if (sa->sa_family != dev->type)
		return -EINVAL;
	if (!netif_device_present(dev))
		return -ENODEV;
4838
	err = ops->ndo_set_mac_address(dev, sa);
4839 4840
	if (err)
		return err;
J
Jiri Pirko 已提交
4841
	dev->addr_assign_type = NET_ADDR_SET;
4842
	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4843
	add_device_randomness(dev->dev_addr, dev->addr_len);
4844
	return 0;
L
Linus Torvalds 已提交
4845
}
E
Eric Dumazet 已提交
4846
EXPORT_SYMBOL(dev_set_mac_address);
L
Linus Torvalds 已提交
4847

J
Jiri Pirko 已提交
4848 4849 4850
/**
 *	dev_change_carrier - Change device carrier
 *	@dev: device
4851
 *	@new_carrier: new value
J
Jiri Pirko 已提交
4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866
 *
 *	Change device carrier
 */
int dev_change_carrier(struct net_device *dev, bool new_carrier)
{
	const struct net_device_ops *ops = dev->netdev_ops;

	if (!ops->ndo_change_carrier)
		return -EOPNOTSUPP;
	if (!netif_device_present(dev))
		return -ENODEV;
	return ops->ndo_change_carrier(dev, new_carrier);
}
EXPORT_SYMBOL(dev_change_carrier);

L
Linus Torvalds 已提交
4867 4868
/**
 *	dev_new_index	-	allocate an ifindex
4869
 *	@net: the applicable net namespace
L
Linus Torvalds 已提交
4870 4871 4872 4873 4874
 *
 *	Returns a suitable unique value for a new device interface
 *	number.  The caller must hold the rtnl semaphore or the
 *	dev_base_lock to be sure it remains unique.
 */
4875
static int dev_new_index(struct net *net)
L
Linus Torvalds 已提交
4876
{
4877
	int ifindex = net->ifindex;
L
Linus Torvalds 已提交
4878 4879 4880
	for (;;) {
		if (++ifindex <= 0)
			ifindex = 1;
4881
		if (!__dev_get_by_index(net, ifindex))
4882
			return net->ifindex = ifindex;
L
Linus Torvalds 已提交
4883 4884 4885 4886
	}
}

/* Delayed registration/unregisteration */
4887
static LIST_HEAD(net_todo_list);
L
Linus Torvalds 已提交
4888

4889
static void net_set_todo(struct net_device *dev)
L
Linus Torvalds 已提交
4890 4891 4892 4893
{
	list_add_tail(&dev->todo_list, &net_todo_list);
}

4894
static void rollback_registered_many(struct list_head *head)
4895
{
4896
	struct net_device *dev, *tmp;
4897

4898 4899 4900
	BUG_ON(dev_boot_phase);
	ASSERT_RTNL();

4901
	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
4902
		/* Some devices call without registering
4903 4904
		 * for initialization unwind. Remove those
		 * devices and proceed with the remaining.
4905 4906
		 */
		if (dev->reg_state == NETREG_UNINITIALIZED) {
4907 4908
			pr_debug("unregister_netdevice: device %s/%p never was registered\n",
				 dev->name, dev);
4909

4910
			WARN_ON(1);
4911 4912
			list_del(&dev->unreg_list);
			continue;
4913
		}
4914
		dev->dismantle = true;
4915
		BUG_ON(dev->reg_state != NETREG_REGISTERED);
4916
	}
4917

4918 4919
	/* If device is running, close it first. */
	dev_close_many(head);
4920

4921
	list_for_each_entry(dev, head, unreg_list) {
4922 4923
		/* And unlink it from device chain. */
		unlist_netdevice(dev);
4924

4925 4926
		dev->reg_state = NETREG_UNREGISTERING;
	}
4927 4928 4929

	synchronize_net();

4930 4931 4932
	list_for_each_entry(dev, head, unreg_list) {
		/* Shutdown queueing discipline. */
		dev_shutdown(dev);
4933 4934


4935 4936 4937 4938
		/* Notify protocols, that we are about to destroy
		   this device. They should clean all the things.
		*/
		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4939

4940 4941 4942 4943
		if (!dev->rtnl_link_ops ||
		    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
			rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);

4944 4945 4946
		/*
		 *	Flush the unicast and multicast chains
		 */
4947
		dev_uc_flush(dev);
4948
		dev_mc_flush(dev);
4949

4950 4951
		if (dev->netdev_ops->ndo_uninit)
			dev->netdev_ops->ndo_uninit(dev);
4952

J
Jiri Pirko 已提交
4953 4954
		/* Notifier chain MUST detach us all upper devices. */
		WARN_ON(netdev_has_any_upper_dev(dev));
4955

4956 4957
		/* Remove entries from kobject tree */
		netdev_unregister_kobject(dev);
4958 4959 4960 4961
#ifdef CONFIG_XPS
		/* Remove XPS queueing entries */
		netif_reset_xps_queues_gt(dev, 0);
#endif
4962
	}
4963

4964
	synchronize_net();
4965

4966
	list_for_each_entry(dev, head, unreg_list)
4967 4968 4969 4970 4971 4972 4973 4974 4975
		dev_put(dev);
}

static void rollback_registered(struct net_device *dev)
{
	LIST_HEAD(single);

	list_add(&dev->unreg_list, &single);
	rollback_registered_many(&single);
E
Eric Dumazet 已提交
4976
	list_del(&single);
4977 4978
}

4979 4980
static netdev_features_t netdev_fix_features(struct net_device *dev,
	netdev_features_t features)
4981
{
4982 4983 4984
	/* Fix illegal checksum combinations */
	if ((features & NETIF_F_HW_CSUM) &&
	    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4985
		netdev_warn(dev, "mixed HW and IP checksum settings.\n");
4986 4987 4988
		features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
	}

4989
	/* TSO requires that SG is present as well. */
4990
	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
4991
		netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
4992
		features &= ~NETIF_F_ALL_TSO;
4993 4994
	}

4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007
	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
					!(features & NETIF_F_IP_CSUM)) {
		netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
		features &= ~NETIF_F_TSO;
		features &= ~NETIF_F_TSO_ECN;
	}

	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
					 !(features & NETIF_F_IPV6_CSUM)) {
		netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
		features &= ~NETIF_F_TSO6;
	}

5008 5009 5010 5011
	/* TSO ECN requires that TSO is present as well. */
	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
		features &= ~NETIF_F_TSO_ECN;

5012 5013
	/* Software GSO depends on SG. */
	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
5014
		netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
5015 5016 5017
		features &= ~NETIF_F_GSO;
	}

5018
	/* UFO needs SG and checksumming */
5019
	if (features & NETIF_F_UFO) {
5020 5021 5022 5023
		/* maybe split UFO into V4 and V6? */
		if (!((features & NETIF_F_GEN_CSUM) ||
		    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
			    == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5024
			netdev_dbg(dev,
5025
				"Dropping NETIF_F_UFO since no checksum offload features.\n");
5026 5027 5028 5029
			features &= ~NETIF_F_UFO;
		}

		if (!(features & NETIF_F_SG)) {
5030
			netdev_dbg(dev,
5031
				"Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
5032 5033 5034 5035 5036 5037 5038
			features &= ~NETIF_F_UFO;
		}
	}

	return features;
}

5039
int __netdev_update_features(struct net_device *dev)
5040
{
5041
	netdev_features_t features;
5042 5043
	int err = 0;

5044 5045
	ASSERT_RTNL();

5046 5047 5048 5049 5050 5051 5052 5053 5054
	features = netdev_get_wanted_features(dev);

	if (dev->netdev_ops->ndo_fix_features)
		features = dev->netdev_ops->ndo_fix_features(dev, features);

	/* driver might be less strict about feature dependencies */
	features = netdev_fix_features(dev, features);

	if (dev->features == features)
5055
		return 0;
5056

5057 5058
	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
		&dev->features, &features);
5059 5060 5061 5062

	if (dev->netdev_ops->ndo_set_features)
		err = dev->netdev_ops->ndo_set_features(dev, features);

5063
	if (unlikely(err < 0)) {
5064
		netdev_err(dev,
5065 5066
			"set_features() failed (%d); wanted %pNF, left %pNF\n",
			err, &features, &dev->features);
5067 5068 5069 5070 5071 5072 5073 5074 5075
		return -1;
	}

	if (!err)
		dev->features = features;

	return 1;
}

5076 5077 5078 5079 5080 5081 5082 5083
/**
 *	netdev_update_features - recalculate device features
 *	@dev: the device to check
 *
 *	Recalculate dev->features set and send notifications if it
 *	has changed. Should be called after driver or hardware dependent
 *	conditions might have changed that influence the features.
 */
5084 5085 5086 5087
void netdev_update_features(struct net_device *dev)
{
	if (__netdev_update_features(dev))
		netdev_features_change(dev);
5088 5089 5090
}
EXPORT_SYMBOL(netdev_update_features);

5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107
/**
 *	netdev_change_features - recalculate device features
 *	@dev: the device to check
 *
 *	Recalculate dev->features set and send notifications even
 *	if they have not changed. Should be called instead of
 *	netdev_update_features() if also dev->vlan_features might
 *	have changed to allow the changes to be propagated to stacked
 *	VLAN devices.
 */
void netdev_change_features(struct net_device *dev)
{
	__netdev_update_features(dev);
	netdev_features_change(dev);
}
EXPORT_SYMBOL(netdev_change_features);

5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134
/**
 *	netif_stacked_transfer_operstate -	transfer operstate
 *	@rootdev: the root or lower level device to transfer state from
 *	@dev: the device to transfer operstate to
 *
 *	Transfer operational state from root to device. This is normally
 *	called when a stacking relationship exists between the root
 *	device and the device(a leaf device).
 */
void netif_stacked_transfer_operstate(const struct net_device *rootdev,
					struct net_device *dev)
{
	if (rootdev->operstate == IF_OPER_DORMANT)
		netif_dormant_on(dev);
	else
		netif_dormant_off(dev);

	if (netif_carrier_ok(rootdev)) {
		if (!netif_carrier_ok(dev))
			netif_carrier_on(dev);
	} else {
		if (netif_carrier_ok(dev))
			netif_carrier_off(dev);
	}
}
EXPORT_SYMBOL(netif_stacked_transfer_operstate);

T
Tom Herbert 已提交
5135
#ifdef CONFIG_RPS
5136 5137 5138
static int netif_alloc_rx_queues(struct net_device *dev)
{
	unsigned int i, count = dev->num_rx_queues;
T
Tom Herbert 已提交
5139
	struct netdev_rx_queue *rx;
5140

T
Tom Herbert 已提交
5141
	BUG_ON(count < 1);
5142

T
Tom Herbert 已提交
5143
	rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5144
	if (!rx)
T
Tom Herbert 已提交
5145
		return -ENOMEM;
5146

T
Tom Herbert 已提交
5147 5148 5149
	dev->_rx = rx;

	for (i = 0; i < count; i++)
T
Tom Herbert 已提交
5150
		rx[i].dev = dev;
5151 5152
	return 0;
}
T
Tom Herbert 已提交
5153
#endif
5154

C
Changli Gao 已提交
5155 5156 5157 5158 5159 5160 5161
static void netdev_init_one_queue(struct net_device *dev,
				  struct netdev_queue *queue, void *_unused)
{
	/* Initialize queue lock */
	spin_lock_init(&queue->_xmit_lock);
	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
	queue->xmit_lock_owner = -1;
5162
	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
C
Changli Gao 已提交
5163
	queue->dev = dev;
T
Tom Herbert 已提交
5164 5165 5166
#ifdef CONFIG_BQL
	dql_init(&queue->dql, HZ);
#endif
C
Changli Gao 已提交
5167 5168
}

5169 5170 5171 5172 5173 5174 5175 5176
static int netif_alloc_netdev_queues(struct net_device *dev)
{
	unsigned int count = dev->num_tx_queues;
	struct netdev_queue *tx;

	BUG_ON(count < 1);

	tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5177
	if (!tx)
5178
		return -ENOMEM;
5179

5180
	dev->_tx = tx;
T
Tom Herbert 已提交
5181

5182 5183
	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
	spin_lock_init(&dev->tx_global_lock);
C
Changli Gao 已提交
5184 5185

	return 0;
5186 5187
}

L
Linus Torvalds 已提交
5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207
/**
 *	register_netdevice	- register a network device
 *	@dev: device to register
 *
 *	Take a completed network device structure and add it to the kernel
 *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
 *	chain. 0 is returned on success. A negative errno code is returned
 *	on a failure to set up the device, or if the name is a duplicate.
 *
 *	Callers must hold the rtnl semaphore. You may want
 *	register_netdev() instead of this.
 *
 *	BUGS:
 *	The locking appears insufficient to guarantee two parallel registers
 *	will not get the same name.
 */

int register_netdevice(struct net_device *dev)
{
	int ret;
5208
	struct net *net = dev_net(dev);
L
Linus Torvalds 已提交
5209 5210 5211 5212

	BUG_ON(dev_boot_phase);
	ASSERT_RTNL();

5213 5214
	might_sleep();

L
Linus Torvalds 已提交
5215 5216
	/* When net_device's are persistent, this will be fatal. */
	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5217
	BUG_ON(!net);
L
Linus Torvalds 已提交
5218

5219
	spin_lock_init(&dev->addr_list_lock);
5220
	netdev_set_addr_lockdep_class(dev);
L
Linus Torvalds 已提交
5221 5222 5223

	dev->iflink = -1;

5224
	ret = dev_get_valid_name(net, dev, dev->name);
5225 5226 5227
	if (ret < 0)
		goto out;

L
Linus Torvalds 已提交
5228
	/* Init, if this function is available */
5229 5230
	if (dev->netdev_ops->ndo_init) {
		ret = dev->netdev_ops->ndo_init(dev);
L
Linus Torvalds 已提交
5231 5232 5233
		if (ret) {
			if (ret > 0)
				ret = -EIO;
5234
			goto out;
L
Linus Torvalds 已提交
5235 5236
		}
	}
5237

5238 5239
	if (((dev->hw_features | dev->features) &
	     NETIF_F_HW_VLAN_CTAG_FILTER) &&
5240 5241 5242 5243 5244 5245 5246
	    (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
	     !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
		netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
		ret = -EINVAL;
		goto err_uninit;
	}

5247 5248 5249 5250 5251 5252
	ret = -EBUSY;
	if (!dev->ifindex)
		dev->ifindex = dev_new_index(net);
	else if (__dev_get_by_index(net, dev->ifindex))
		goto err_uninit;

L
Linus Torvalds 已提交
5253 5254 5255
	if (dev->iflink == -1)
		dev->iflink = dev->ifindex;

5256 5257 5258 5259
	/* Transfer changeable features to wanted_features and enable
	 * software offloads (GSO and GRO).
	 */
	dev->hw_features |= NETIF_F_SOFT_FEATURES;
5260 5261
	dev->features |= NETIF_F_SOFT_FEATURES;
	dev->wanted_features = dev->features & dev->hw_features;
L
Linus Torvalds 已提交
5262

5263
	/* Turn on no cache copy if HW is doing checksum */
5264 5265 5266 5267 5268 5269
	if (!(dev->flags & IFF_LOOPBACK)) {
		dev->hw_features |= NETIF_F_NOCACHE_COPY;
		if (dev->features & NETIF_F_ALL_CSUM) {
			dev->wanted_features |= NETIF_F_NOCACHE_COPY;
			dev->features |= NETIF_F_NOCACHE_COPY;
		}
5270 5271
	}

5272
	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
5273
	 */
5274
	dev->vlan_features |= NETIF_F_HIGHDMA;
5275

5276 5277 5278 5279
	/* Make NETIF_F_SG inheritable to tunnel devices.
	 */
	dev->hw_enc_features |= NETIF_F_SG;

S
Simon Horman 已提交
5280 5281 5282 5283
	/* Make NETIF_F_SG inheritable to MPLS.
	 */
	dev->mpls_features |= NETIF_F_SG;

5284 5285 5286 5287 5288
	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
	ret = notifier_to_errno(ret);
	if (ret)
		goto err_uninit;

5289
	ret = netdev_register_kobject(dev);
5290
	if (ret)
5291
		goto err_uninit;
5292 5293
	dev->reg_state = NETREG_REGISTERED;

5294
	__netdev_update_features(dev);
5295

L
Linus Torvalds 已提交
5296 5297 5298 5299 5300 5301 5302
	/*
	 *	Default initial state at registry is that the
	 *	device is present.
	 */

	set_bit(__LINK_STATE_PRESENT, &dev->state);

5303 5304
	linkwatch_init_dev(dev);

L
Linus Torvalds 已提交
5305 5306
	dev_init_scheduler(dev);
	dev_hold(dev);
5307
	list_netdevice(dev);
5308
	add_device_randomness(dev->dev_addr, dev->addr_len);
L
Linus Torvalds 已提交
5309

5310 5311 5312 5313 5314 5315 5316
	/* If the device has permanent device address, driver should
	 * set dev_addr and also addr_assign_type should be set to
	 * NET_ADDR_PERM (default value).
	 */
	if (dev->addr_assign_type == NET_ADDR_PERM)
		memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);

L
Linus Torvalds 已提交
5317
	/* Notify protocols, that a new device appeared. */
5318
	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5319
	ret = notifier_to_errno(ret);
5320 5321 5322 5323
	if (ret) {
		rollback_registered(dev);
		dev->reg_state = NETREG_UNREGISTERED;
	}
5324 5325 5326 5327
	/*
	 *	Prevent userspace races by waiting until the network
	 *	device is fully setup before sending notifications.
	 */
5328 5329 5330
	if (!dev->rtnl_link_ops ||
	    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
L
Linus Torvalds 已提交
5331 5332 5333

out:
	return ret;
5334 5335

err_uninit:
5336 5337
	if (dev->netdev_ops->ndo_uninit)
		dev->netdev_ops->ndo_uninit(dev);
5338
	goto out;
L
Linus Torvalds 已提交
5339
}
E
Eric Dumazet 已提交
5340
EXPORT_SYMBOL(register_netdevice);
L
Linus Torvalds 已提交
5341

5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372
/**
 *	init_dummy_netdev	- init a dummy network device for NAPI
 *	@dev: device to init
 *
 *	This takes a network device structure and initialize the minimum
 *	amount of fields so it can be used to schedule NAPI polls without
 *	registering a full blown interface. This is to be used by drivers
 *	that need to tie several hardware interfaces to a single NAPI
 *	poll scheduler due to HW limitations.
 */
int init_dummy_netdev(struct net_device *dev)
{
	/* Clear everything. Note we don't initialize spinlocks
	 * are they aren't supposed to be taken by any of the
	 * NAPI code and this dummy netdev is supposed to be
	 * only ever used for NAPI polls
	 */
	memset(dev, 0, sizeof(struct net_device));

	/* make sure we BUG if trying to hit standard
	 * register/unregister code path
	 */
	dev->reg_state = NETREG_DUMMY;

	/* NAPI wants this */
	INIT_LIST_HEAD(&dev->napi_list);

	/* a dummy interface is started by default */
	set_bit(__LINK_STATE_PRESENT, &dev->state);
	set_bit(__LINK_STATE_START, &dev->state);

E
Eric Dumazet 已提交
5373 5374 5375 5376 5377
	/* Note : We dont allocate pcpu_refcnt for dummy devices,
	 * because users of this 'device' dont need to change
	 * its refcount.
	 */

5378 5379 5380 5381 5382
	return 0;
}
EXPORT_SYMBOL_GPL(init_dummy_netdev);


L
Linus Torvalds 已提交
5383 5384 5385 5386 5387 5388 5389 5390 5391
/**
 *	register_netdev	- register a network device
 *	@dev: device to register
 *
 *	Take a completed network device structure and add it to the kernel
 *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
 *	chain. 0 is returned on success. A negative errno code is returned
 *	on a failure to set up the device, or if the name is a duplicate.
 *
5392
 *	This is a wrapper around register_netdevice that takes the rtnl semaphore
L
Linus Torvalds 已提交
5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406
 *	and expands the device name if you passed a format string to
 *	alloc_netdev.
 */
int register_netdev(struct net_device *dev)
{
	int err;

	rtnl_lock();
	err = register_netdevice(dev);
	rtnl_unlock();
	return err;
}
EXPORT_SYMBOL(register_netdev);

E
Eric Dumazet 已提交
5407 5408 5409 5410 5411 5412 5413 5414 5415 5416
int netdev_refcnt_read(const struct net_device *dev)
{
	int i, refcnt = 0;

	for_each_possible_cpu(i)
		refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
	return refcnt;
}
EXPORT_SYMBOL(netdev_refcnt_read);

5417
/**
L
Linus Torvalds 已提交
5418
 * netdev_wait_allrefs - wait until all references are gone.
5419
 * @dev: target net_device
L
Linus Torvalds 已提交
5420 5421 5422 5423 5424 5425 5426
 *
 * This is called when unregistering network devices.
 *
 * Any protocol or device that holds a reference should register
 * for netdevice notification, and cleanup and put back the
 * reference if they receive an UNREGISTER event.
 * We can get stuck here if buggy protocols don't correctly
5427
 * call dev_put.
L
Linus Torvalds 已提交
5428 5429 5430 5431
 */
static void netdev_wait_allrefs(struct net_device *dev)
{
	unsigned long rebroadcast_time, warning_time;
E
Eric Dumazet 已提交
5432
	int refcnt;
L
Linus Torvalds 已提交
5433

5434 5435
	linkwatch_forget_dev(dev);

L
Linus Torvalds 已提交
5436
	rebroadcast_time = warning_time = jiffies;
E
Eric Dumazet 已提交
5437 5438 5439
	refcnt = netdev_refcnt_read(dev);

	while (refcnt != 0) {
L
Linus Torvalds 已提交
5440
		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5441
			rtnl_lock();
L
Linus Torvalds 已提交
5442 5443

			/* Rebroadcast unregister notification */
5444
			call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
L
Linus Torvalds 已提交
5445

5446
			__rtnl_unlock();
5447
			rcu_barrier();
5448 5449
			rtnl_lock();

5450
			call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
L
Linus Torvalds 已提交
5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461
			if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
				     &dev->state)) {
				/* We must not have linkwatch events
				 * pending on unregister. If this
				 * happens, we simply run the queue
				 * unscheduled, resulting in a noop
				 * for this device.
				 */
				linkwatch_run_queue();
			}

5462
			__rtnl_unlock();
L
Linus Torvalds 已提交
5463 5464 5465 5466 5467 5468

			rebroadcast_time = jiffies;
		}

		msleep(250);

E
Eric Dumazet 已提交
5469 5470
		refcnt = netdev_refcnt_read(dev);

L
Linus Torvalds 已提交
5471
		if (time_after(jiffies, warning_time + 10 * HZ)) {
5472 5473
			pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
				 dev->name, refcnt);
L
Linus Torvalds 已提交
5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492
			warning_time = jiffies;
		}
	}
}

/* The sequence is:
 *
 *	rtnl_lock();
 *	...
 *	register_netdevice(x1);
 *	register_netdevice(x2);
 *	...
 *	unregister_netdevice(y1);
 *	unregister_netdevice(y2);
 *      ...
 *	rtnl_unlock();
 *	free_netdev(y1);
 *	free_netdev(y2);
 *
H
Herbert Xu 已提交
5493
 * We are invoked by rtnl_unlock().
L
Linus Torvalds 已提交
5494
 * This allows us to deal with problems:
5495
 * 1) We can delete sysfs objects which invoke hotplug
L
Linus Torvalds 已提交
5496 5497 5498
 *    without deadlocking with linkwatch via keventd.
 * 2) Since we run with the RTNL semaphore not held, we can sleep
 *    safely in order to wait for the netdev refcnt to drop to zero.
H
Herbert Xu 已提交
5499 5500 5501
 *
 * We must not return until all unregister events added during
 * the interval the lock was held have been completed.
L
Linus Torvalds 已提交
5502 5503 5504
 */
void netdev_run_todo(void)
{
5505
	struct list_head list;
L
Linus Torvalds 已提交
5506 5507

	/* Snapshot list, allow later requests */
5508
	list_replace_init(&net_todo_list, &list);
H
Herbert Xu 已提交
5509 5510

	__rtnl_unlock();
5511

5512 5513

	/* Wait for rcu callbacks to finish before next phase */
5514 5515 5516
	if (!list_empty(&list))
		rcu_barrier();

L
Linus Torvalds 已提交
5517 5518
	while (!list_empty(&list)) {
		struct net_device *dev
5519
			= list_first_entry(&list, struct net_device, todo_list);
L
Linus Torvalds 已提交
5520 5521
		list_del(&dev->todo_list);

5522
		rtnl_lock();
5523
		call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5524
		__rtnl_unlock();
5525

5526
		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5527
			pr_err("network todo '%s' but state %d\n",
5528 5529 5530 5531
			       dev->name, dev->reg_state);
			dump_stack();
			continue;
		}
L
Linus Torvalds 已提交
5532

5533
		dev->reg_state = NETREG_UNREGISTERED;
L
Linus Torvalds 已提交
5534

5535
		on_each_cpu(flush_backlog, dev, 1);
5536

5537
		netdev_wait_allrefs(dev);
L
Linus Torvalds 已提交
5538

5539
		/* paranoia */
E
Eric Dumazet 已提交
5540
		BUG_ON(netdev_refcnt_read(dev));
5541 5542
		WARN_ON(rcu_access_pointer(dev->ip_ptr));
		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
5543
		WARN_ON(dev->dn_ptr);
L
Linus Torvalds 已提交
5544

5545 5546
		if (dev->destructor)
			dev->destructor(dev);
5547 5548 5549

		/* Free network device */
		kobject_put(&dev->dev.kobj);
L
Linus Torvalds 已提交
5550 5551 5552
	}
}

5553 5554 5555
/* Convert net_device_stats to rtnl_link_stats64.  They have the same
 * fields in the same order, with only the type differing.
 */
5556 5557
void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
			     const struct net_device_stats *netdev_stats)
5558 5559
{
#if BITS_PER_LONG == 64
5560 5561
	BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
	memcpy(stats64, netdev_stats, sizeof(*stats64));
5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572
#else
	size_t i, n = sizeof(*stats64) / sizeof(u64);
	const unsigned long *src = (const unsigned long *)netdev_stats;
	u64 *dst = (u64 *)stats64;

	BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
		     sizeof(*stats64) / sizeof(u64));
	for (i = 0; i < n; i++)
		dst[i] = src[i];
#endif
}
5573
EXPORT_SYMBOL(netdev_stats_to_stats64);
5574

5575 5576 5577
/**
 *	dev_get_stats	- get network device statistics
 *	@dev: device to get statistics from
5578
 *	@storage: place to store stats
5579
 *
5580 5581 5582 5583
 *	Get network statistics from device. Return @storage.
 *	The device driver may provide its own method by setting
 *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
 *	otherwise the internal statistics structure is used.
5584
 */
5585 5586
struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
					struct rtnl_link_stats64 *storage)
5587
{
5588 5589
	const struct net_device_ops *ops = dev->netdev_ops;

5590 5591
	if (ops->ndo_get_stats64) {
		memset(storage, 0, sizeof(*storage));
5592 5593
		ops->ndo_get_stats64(dev, storage);
	} else if (ops->ndo_get_stats) {
5594
		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
5595 5596
	} else {
		netdev_stats_to_stats64(storage, &dev->stats);
5597
	}
5598
	storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
5599
	return storage;
R
Rusty Russell 已提交
5600
}
5601
EXPORT_SYMBOL(dev_get_stats);
R
Rusty Russell 已提交
5602

5603
struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
5604
{
5605
	struct netdev_queue *queue = dev_ingress_queue(dev);
5606

5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618
#ifdef CONFIG_NET_CLS_ACT
	if (queue)
		return queue;
	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
	if (!queue)
		return NULL;
	netdev_init_one_queue(dev, queue, NULL);
	queue->qdisc = &noop_qdisc;
	queue->qdisc_sleeping = &noop_qdisc;
	rcu_assign_pointer(dev->ingress_queue, queue);
#endif
	return queue;
5619 5620
}

5621 5622
static const struct ethtool_ops default_ethtool_ops;

5623 5624 5625 5626 5627 5628 5629 5630
void netdev_set_default_ethtool_ops(struct net_device *dev,
				    const struct ethtool_ops *ops)
{
	if (dev->ethtool_ops == &default_ethtool_ops)
		dev->ethtool_ops = ops;
}
EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);

L
Linus Torvalds 已提交
5631
/**
T
Tom Herbert 已提交
5632
 *	alloc_netdev_mqs - allocate network device
L
Linus Torvalds 已提交
5633 5634 5635
 *	@sizeof_priv:	size of private data to allocate space for
 *	@name:		device name format string
 *	@setup:		callback to initialize device
T
Tom Herbert 已提交
5636 5637
 *	@txqs:		the number of TX subqueues to allocate
 *	@rxqs:		the number of RX subqueues to allocate
L
Linus Torvalds 已提交
5638 5639
 *
 *	Allocates a struct net_device with private data area for driver use
5640
 *	and performs basic initialization.  Also allocates subquue structs
T
Tom Herbert 已提交
5641
 *	for each queue on the device.
L
Linus Torvalds 已提交
5642
 */
T
Tom Herbert 已提交
5643 5644 5645
struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
		void (*setup)(struct net_device *),
		unsigned int txqs, unsigned int rxqs)
L
Linus Torvalds 已提交
5646 5647
{
	struct net_device *dev;
5648
	size_t alloc_size;
5649
	struct net_device *p;
L
Linus Torvalds 已提交
5650

5651 5652
	BUG_ON(strlen(name) >= sizeof(dev->name));

T
Tom Herbert 已提交
5653
	if (txqs < 1) {
5654
		pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
5655 5656 5657
		return NULL;
	}

T
Tom Herbert 已提交
5658 5659
#ifdef CONFIG_RPS
	if (rxqs < 1) {
5660
		pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
T
Tom Herbert 已提交
5661 5662 5663 5664
		return NULL;
	}
#endif

5665
	alloc_size = sizeof(struct net_device);
5666 5667
	if (sizeof_priv) {
		/* ensure 32-byte alignment of private area */
5668
		alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5669 5670 5671
		alloc_size += sizeof_priv;
	}
	/* ensure 32-byte alignment of whole construct */
5672
	alloc_size += NETDEV_ALIGN - 1;
L
Linus Torvalds 已提交
5673

5674
	p = kzalloc(alloc_size, GFP_KERNEL);
5675
	if (!p)
L
Linus Torvalds 已提交
5676 5677
		return NULL;

5678
	dev = PTR_ALIGN(p, NETDEV_ALIGN);
L
Linus Torvalds 已提交
5679
	dev->padded = (char *)dev - (char *)p;
5680

E
Eric Dumazet 已提交
5681 5682
	dev->pcpu_refcnt = alloc_percpu(int);
	if (!dev->pcpu_refcnt)
5683
		goto free_p;
5684 5685

	if (dev_addr_init(dev))
E
Eric Dumazet 已提交
5686
		goto free_pcpu;
5687

5688
	dev_mc_init(dev);
5689
	dev_uc_init(dev);
J
Jiri Pirko 已提交
5690

5691
	dev_net_set(dev, &init_net);
L
Linus Torvalds 已提交
5692

5693
	dev->gso_max_size = GSO_MAX_SIZE;
5694
	dev->gso_max_segs = GSO_MAX_SEGS;
5695 5696 5697 5698

	INIT_LIST_HEAD(&dev->napi_list);
	INIT_LIST_HEAD(&dev->unreg_list);
	INIT_LIST_HEAD(&dev->link_watch_list);
J
Jiri Pirko 已提交
5699
	INIT_LIST_HEAD(&dev->upper_dev_list);
5700 5701 5702
	dev->priv_flags = IFF_XMIT_DST_RELEASE;
	setup(dev);

T
Tom Herbert 已提交
5703 5704
	dev->num_tx_queues = txqs;
	dev->real_num_tx_queues = txqs;
5705
	if (netif_alloc_netdev_queues(dev))
5706
		goto free_all;
5707

E
Eric Dumazet 已提交
5708
#ifdef CONFIG_RPS
T
Tom Herbert 已提交
5709 5710
	dev->num_rx_queues = rxqs;
	dev->real_num_rx_queues = rxqs;
T
Tom Herbert 已提交
5711
	if (netif_alloc_rx_queues(dev))
5712
		goto free_all;
E
Eric Dumazet 已提交
5713
#endif
T
Tom Herbert 已提交
5714

L
Linus Torvalds 已提交
5715
	strcpy(dev->name, name);
5716
	dev->group = INIT_NETDEV_GROUP;
5717 5718
	if (!dev->ethtool_ops)
		dev->ethtool_ops = &default_ethtool_ops;
L
Linus Torvalds 已提交
5719
	return dev;
5720

5721 5722 5723 5724
free_all:
	free_netdev(dev);
	return NULL;

E
Eric Dumazet 已提交
5725 5726
free_pcpu:
	free_percpu(dev->pcpu_refcnt);
5727
	kfree(dev->_tx);
T
Tom Herbert 已提交
5728 5729 5730 5731
#ifdef CONFIG_RPS
	kfree(dev->_rx);
#endif

5732 5733 5734
free_p:
	kfree(p);
	return NULL;
L
Linus Torvalds 已提交
5735
}
T
Tom Herbert 已提交
5736
EXPORT_SYMBOL(alloc_netdev_mqs);
L
Linus Torvalds 已提交
5737 5738 5739 5740 5741

/**
 *	free_netdev - free network device
 *	@dev: device
 *
5742 5743
 *	This function does the last stage of destroying an allocated device
 * 	interface. The reference to the device object is released.
L
Linus Torvalds 已提交
5744 5745 5746 5747
 *	If this is the last reference then it will be freed.
 */
void free_netdev(struct net_device *dev)
{
5748 5749
	struct napi_struct *p, *n;

5750 5751
	release_net(dev_net(dev));

5752
	kfree(dev->_tx);
T
Tom Herbert 已提交
5753 5754 5755
#ifdef CONFIG_RPS
	kfree(dev->_rx);
#endif
5756

5757
	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
5758

5759 5760 5761
	/* Flush device addresses */
	dev_addr_flush(dev);

5762 5763 5764
	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
		netif_napi_del(p);

E
Eric Dumazet 已提交
5765 5766 5767
	free_percpu(dev->pcpu_refcnt);
	dev->pcpu_refcnt = NULL;

S
Stephen Hemminger 已提交
5768
	/*  Compatibility with error handling in drivers */
L
Linus Torvalds 已提交
5769 5770 5771 5772 5773 5774 5775 5776
	if (dev->reg_state == NETREG_UNINITIALIZED) {
		kfree((char *)dev - dev->padded);
		return;
	}

	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
	dev->reg_state = NETREG_RELEASED;

5777 5778
	/* will free via device release */
	put_device(&dev->dev);
L
Linus Torvalds 已提交
5779
}
E
Eric Dumazet 已提交
5780
EXPORT_SYMBOL(free_netdev);
5781

5782 5783 5784 5785 5786 5787
/**
 *	synchronize_net -  Synchronize with packet receive processing
 *
 *	Wait for packets currently being received to be done.
 *	Does not block later packets from starting.
 */
5788
void synchronize_net(void)
L
Linus Torvalds 已提交
5789 5790
{
	might_sleep();
5791 5792 5793 5794
	if (rtnl_is_locked())
		synchronize_rcu_expedited();
	else
		synchronize_rcu();
L
Linus Torvalds 已提交
5795
}
E
Eric Dumazet 已提交
5796
EXPORT_SYMBOL(synchronize_net);
L
Linus Torvalds 已提交
5797 5798

/**
5799
 *	unregister_netdevice_queue - remove device from the kernel
L
Linus Torvalds 已提交
5800
 *	@dev: device
5801
 *	@head: list
5802
 *
L
Linus Torvalds 已提交
5803
 *	This function shuts down a device interface and removes it
5804
 *	from the kernel tables.
5805
 *	If head not NULL, device is queued to be unregistered later.
L
Linus Torvalds 已提交
5806 5807 5808 5809 5810
 *
 *	Callers must hold the rtnl semaphore.  You may want
 *	unregister_netdev() instead of this.
 */

5811
void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
L
Linus Torvalds 已提交
5812
{
5813 5814
	ASSERT_RTNL();

5815
	if (head) {
5816
		list_move_tail(&dev->unreg_list, head);
5817 5818 5819 5820 5821
	} else {
		rollback_registered(dev);
		/* Finish processing unregister after unlock */
		net_set_todo(dev);
	}
L
Linus Torvalds 已提交
5822
}
5823
EXPORT_SYMBOL(unregister_netdevice_queue);
L
Linus Torvalds 已提交
5824

5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838
/**
 *	unregister_netdevice_many - unregister many devices
 *	@head: list of devices
 */
void unregister_netdevice_many(struct list_head *head)
{
	struct net_device *dev;

	if (!list_empty(head)) {
		rollback_registered_many(head);
		list_for_each_entry(dev, head, unreg_list)
			net_set_todo(dev);
	}
}
5839
EXPORT_SYMBOL(unregister_netdevice_many);
5840

L
Linus Torvalds 已提交
5841 5842 5843 5844 5845
/**
 *	unregister_netdev - remove device from the kernel
 *	@dev: device
 *
 *	This function shuts down a device interface and removes it
5846
 *	from the kernel tables.
L
Linus Torvalds 已提交
5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859
 *
 *	This is just a wrapper for unregister_netdevice that takes
 *	the rtnl semaphore.  In general you want to use this and not
 *	unregister_netdevice.
 */
void unregister_netdev(struct net_device *dev)
{
	rtnl_lock();
	unregister_netdevice(dev);
	rtnl_unlock();
}
EXPORT_SYMBOL(unregister_netdev);

5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890
/**
 *	dev_change_net_namespace - move device to different nethost namespace
 *	@dev: device
 *	@net: network namespace
 *	@pat: If not NULL name pattern to try if the current device name
 *	      is already taken in the destination network namespace.
 *
 *	This function shuts down a device interface and moves it
 *	to a new network namespace. On success 0 is returned, on
 *	a failure a netagive errno code is returned.
 *
 *	Callers must hold the rtnl semaphore.
 */

int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
{
	int err;

	ASSERT_RTNL();

	/* Don't allow namespace local devices to be moved. */
	err = -EINVAL;
	if (dev->features & NETIF_F_NETNS_LOCAL)
		goto out;

	/* Ensure the device has been registrered */
	if (dev->reg_state != NETREG_REGISTERED)
		goto out;

	/* Get out if there is nothing todo */
	err = 0;
5891
	if (net_eq(dev_net(dev), net))
5892 5893 5894 5895 5896 5897
		goto out;

	/* Pick the destination device name, and ensure
	 * we can use it in the destination network namespace.
	 */
	err = -EEXIST;
5898
	if (__dev_get_by_name(net, dev->name)) {
5899 5900 5901
		/* We get here if we can't use the current device name */
		if (!pat)
			goto out;
5902
		if (dev_get_valid_name(net, dev, pat) < 0)
5903 5904 5905 5906 5907 5908 5909 5910
			goto out;
	}

	/*
	 * And now a mini version of register_netdevice unregister_netdevice.
	 */

	/* If device is running close it first. */
5911
	dev_close(dev);
5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923

	/* And unlink it from device chain */
	err = -ENODEV;
	unlist_netdevice(dev);

	synchronize_net();

	/* Shutdown queueing discipline. */
	dev_shutdown(dev);

	/* Notify protocols, that we are about to destroy
	   this device. They should clean all the things.
5924 5925 5926 5927

	   Note that dev->reg_state stays at NETREG_REGISTERED.
	   This is wanted because this way 8021q and macvlan know
	   the device is just moving and can keep their slaves up.
5928 5929
	*/
	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5930 5931
	rcu_barrier();
	call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
5932
	rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5933 5934 5935 5936

	/*
	 *	Flush the unicast and multicast chains
	 */
5937
	dev_uc_flush(dev);
5938
	dev_mc_flush(dev);
5939

5940 5941 5942
	/* Send a netdev-removed uevent to the old namespace */
	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);

5943
	/* Actually switch the network namespace */
5944
	dev_net_set(dev, net);
5945 5946 5947 5948 5949 5950 5951 5952 5953

	/* If there is an ifindex conflict assign a new one */
	if (__dev_get_by_index(net, dev->ifindex)) {
		int iflink = (dev->iflink == dev->ifindex);
		dev->ifindex = dev_new_index(net);
		if (iflink)
			dev->iflink = dev->ifindex;
	}

5954 5955 5956
	/* Send a netdev-add uevent to the new namespace */
	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);

5957
	/* Fixup kobjects */
5958
	err = device_rename(&dev->dev, dev->name);
5959
	WARN_ON(err);
5960 5961 5962 5963 5964 5965 5966

	/* Add the device back in the hashes */
	list_netdevice(dev);

	/* Notify protocols, that a new device appeared. */
	call_netdevice_notifiers(NETDEV_REGISTER, dev);

5967 5968 5969 5970 5971 5972
	/*
	 *	Prevent userspace races by waiting until the network
	 *	device is fully setup before sending notifications.
	 */
	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);

5973 5974 5975 5976 5977
	synchronize_net();
	err = 0;
out:
	return err;
}
5978
EXPORT_SYMBOL_GPL(dev_change_net_namespace);
5979

L
Linus Torvalds 已提交
5980 5981 5982 5983 5984 5985 5986 5987 5988
static int dev_cpu_callback(struct notifier_block *nfb,
			    unsigned long action,
			    void *ocpu)
{
	struct sk_buff **list_skb;
	struct sk_buff *skb;
	unsigned int cpu, oldcpu = (unsigned long)ocpu;
	struct softnet_data *sd, *oldsd;

5989
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
L
Linus Torvalds 已提交
5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005
		return NOTIFY_OK;

	local_irq_disable();
	cpu = smp_processor_id();
	sd = &per_cpu(softnet_data, cpu);
	oldsd = &per_cpu(softnet_data, oldcpu);

	/* Find end of our completion_queue. */
	list_skb = &sd->completion_queue;
	while (*list_skb)
		list_skb = &(*list_skb)->next;
	/* Append completion queue from offline CPU. */
	*list_skb = oldsd->completion_queue;
	oldsd->completion_queue = NULL;

	/* Append output queue from offline CPU. */
6006 6007 6008 6009 6010 6011
	if (oldsd->output_queue) {
		*sd->output_queue_tailp = oldsd->output_queue;
		sd->output_queue_tailp = oldsd->output_queue_tailp;
		oldsd->output_queue = NULL;
		oldsd->output_queue_tailp = &oldsd->output_queue;
	}
6012 6013 6014 6015 6016
	/* Append NAPI poll list from offline CPU. */
	if (!list_empty(&oldsd->poll_list)) {
		list_splice_init(&oldsd->poll_list, &sd->poll_list);
		raise_softirq_irqoff(NET_RX_SOFTIRQ);
	}
L
Linus Torvalds 已提交
6017 6018 6019 6020 6021

	raise_softirq_irqoff(NET_TX_SOFTIRQ);
	local_irq_enable();

	/* Process offline CPU's input_pkt_queue */
6022
	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
L
Linus Torvalds 已提交
6023
		netif_rx(skb);
6024
		input_queue_head_incr(oldsd);
T
Tom Herbert 已提交
6025
	}
6026
	while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
6027
		netif_rx(skb);
6028 6029
		input_queue_head_incr(oldsd);
	}
L
Linus Torvalds 已提交
6030 6031 6032 6033 6034

	return NOTIFY_OK;
}


6035
/**
6036 6037 6038 6039
 *	netdev_increment_features - increment feature set by one
 *	@all: current feature set
 *	@one: new feature set
 *	@mask: mask feature set
6040 6041
 *
 *	Computes a new feature set after adding a device with feature set
6042 6043
 *	@one to the master device with current feature set @all.  Will not
 *	enable anything that is off in @mask. Returns the new feature set.
6044
 */
6045 6046
netdev_features_t netdev_increment_features(netdev_features_t all,
	netdev_features_t one, netdev_features_t mask)
6047
{
6048 6049 6050
	if (mask & NETIF_F_GEN_CSUM)
		mask |= NETIF_F_ALL_CSUM;
	mask |= NETIF_F_VLAN_CHALLENGED;
6051

6052 6053
	all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
	all &= one | ~NETIF_F_ALL_FOR_ALL;
6054

6055 6056 6057
	/* If one device supports hw checksumming, set for all. */
	if (all & NETIF_F_GEN_CSUM)
		all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
6058 6059 6060

	return all;
}
6061
EXPORT_SYMBOL(netdev_increment_features);
6062

6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075
static struct hlist_head *netdev_create_hash(void)
{
	int i;
	struct hlist_head *hash;

	hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
	if (hash != NULL)
		for (i = 0; i < NETDEV_HASHENTRIES; i++)
			INIT_HLIST_HEAD(&hash[i]);

	return hash;
}

6076
/* Initialize per network namespace state */
6077
static int __net_init netdev_init(struct net *net)
6078
{
6079 6080
	if (net != &init_net)
		INIT_LIST_HEAD(&net->dev_base_head);
6081

6082 6083 6084
	net->dev_name_head = netdev_create_hash();
	if (net->dev_name_head == NULL)
		goto err_name;
6085

6086 6087 6088
	net->dev_index_head = netdev_create_hash();
	if (net->dev_index_head == NULL)
		goto err_idx;
6089 6090

	return 0;
6091 6092 6093 6094 6095

err_idx:
	kfree(net->dev_name_head);
err_name:
	return -ENOMEM;
6096 6097
}

6098 6099 6100 6101 6102 6103
/**
 *	netdev_drivername - network driver for the device
 *	@dev: network device
 *
 *	Determine network driver for device.
 */
6104
const char *netdev_drivername(const struct net_device *dev)
6105
{
6106 6107
	const struct device_driver *driver;
	const struct device *parent;
6108
	const char *empty = "";
6109 6110 6111

	parent = dev->dev.parent;
	if (!parent)
6112
		return empty;
6113 6114 6115

	driver = parent->driver;
	if (driver && driver->name)
6116 6117
		return driver->name;
	return empty;
6118 6119
}

6120
static int __netdev_printk(const char *level, const struct net_device *dev,
6121 6122 6123 6124
			   struct va_format *vaf)
{
	int r;

6125
	if (dev && dev->dev.parent) {
6126 6127 6128 6129 6130 6131
		r = dev_printk_emit(level[1] - '0',
				    dev->dev.parent,
				    "%s %s %s: %pV",
				    dev_driver_string(dev->dev.parent),
				    dev_name(dev->dev.parent),
				    netdev_name(dev), vaf);
6132
	} else if (dev) {
6133
		r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6134
	} else {
6135
		r = printk("%s(NULL net_device): %pV", level, vaf);
6136
	}
6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153

	return r;
}

int netdev_printk(const char *level, const struct net_device *dev,
		  const char *format, ...)
{
	struct va_format vaf;
	va_list args;
	int r;

	va_start(args, format);

	vaf.fmt = format;
	vaf.va = &args;

	r = __netdev_printk(level, dev, &vaf);
6154

6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
	va_end(args);

	return r;
}
EXPORT_SYMBOL(netdev_printk);

#define define_netdev_printk_level(func, level)			\
int func(const struct net_device *dev, const char *fmt, ...)	\
{								\
	int r;							\
	struct va_format vaf;					\
	va_list args;						\
								\
	va_start(args, fmt);					\
								\
	vaf.fmt = fmt;						\
	vaf.va = &args;						\
								\
	r = __netdev_printk(level, dev, &vaf);			\
6174
								\
6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188
	va_end(args);						\
								\
	return r;						\
}								\
EXPORT_SYMBOL(func);

define_netdev_printk_level(netdev_emerg, KERN_EMERG);
define_netdev_printk_level(netdev_alert, KERN_ALERT);
define_netdev_printk_level(netdev_crit, KERN_CRIT);
define_netdev_printk_level(netdev_err, KERN_ERR);
define_netdev_printk_level(netdev_warn, KERN_WARNING);
define_netdev_printk_level(netdev_notice, KERN_NOTICE);
define_netdev_printk_level(netdev_info, KERN_INFO);

6189
static void __net_exit netdev_exit(struct net *net)
6190 6191 6192 6193 6194
{
	kfree(net->dev_name_head);
	kfree(net->dev_index_head);
}

6195
static struct pernet_operations __net_initdata netdev_net_ops = {
6196 6197 6198 6199
	.init = netdev_init,
	.exit = netdev_exit,
};

6200
static void __net_exit default_device_exit(struct net *net)
6201
{
6202
	struct net_device *dev, *aux;
6203
	/*
6204
	 * Push all migratable network devices back to the
6205 6206 6207
	 * initial network namespace
	 */
	rtnl_lock();
6208
	for_each_netdev_safe(net, dev, aux) {
6209
		int err;
6210
		char fb_name[IFNAMSIZ];
6211 6212 6213 6214 6215

		/* Ignore unmoveable devices (i.e. loopback) */
		if (dev->features & NETIF_F_NETNS_LOCAL)
			continue;

6216 6217 6218
		/* Leave virtual devices for the generic cleanup */
		if (dev->rtnl_link_ops)
			continue;
6219

L
Lucas De Marchi 已提交
6220
		/* Push remaining network devices to init_net */
6221 6222
		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
		err = dev_change_net_namespace(dev, &init_net, fb_name);
6223
		if (err) {
6224 6225
			pr_emerg("%s: failed to move %s to init_net: %d\n",
				 __func__, dev->name, err);
6226
			BUG();
6227 6228 6229 6230 6231
		}
	}
	rtnl_unlock();
}

6232 6233 6234
static void __net_exit default_device_exit_batch(struct list_head *net_list)
{
	/* At exit all network devices most be removed from a network
6235
	 * namespace.  Do this in the reverse order of registration.
6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252
	 * Do this across as many network namespaces as possible to
	 * improve batching efficiency.
	 */
	struct net_device *dev;
	struct net *net;
	LIST_HEAD(dev_kill_list);

	rtnl_lock();
	list_for_each_entry(net, net_list, exit_list) {
		for_each_netdev_reverse(net, dev) {
			if (dev->rtnl_link_ops)
				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
			else
				unregister_netdevice_queue(dev, &dev_kill_list);
		}
	}
	unregister_netdevice_many(&dev_kill_list);
E
Eric Dumazet 已提交
6253
	list_del(&dev_kill_list);
6254 6255 6256
	rtnl_unlock();
}

6257
static struct pernet_operations __net_initdata default_device_ops = {
6258
	.exit = default_device_exit,
6259
	.exit_batch = default_device_exit_batch,
6260 6261
};

L
Linus Torvalds 已提交
6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281
/*
 *	Initialize the DEV module. At boot time this walks the device list and
 *	unhooks any devices that fail to initialise (normally hardware not
 *	present) and leaves us with a valid list of present and active devices.
 *
 */

/*
 *       This is called single threaded during boot, so no need
 *       to take the rtnl semaphore.
 */
static int __init net_dev_init(void)
{
	int i, rc = -ENOMEM;

	BUG_ON(!dev_boot_phase);

	if (dev_proc_init())
		goto out;

6282
	if (netdev_kobject_init())
L
Linus Torvalds 已提交
6283 6284 6285
		goto out;

	INIT_LIST_HEAD(&ptype_all);
6286
	for (i = 0; i < PTYPE_HASH_SIZE; i++)
L
Linus Torvalds 已提交
6287 6288
		INIT_LIST_HEAD(&ptype_base[i]);

6289 6290
	INIT_LIST_HEAD(&offload_base);

6291 6292
	if (register_pernet_subsys(&netdev_net_ops))
		goto out;
L
Linus Torvalds 已提交
6293 6294 6295 6296 6297

	/*
	 *	Initialise the packet receive queues.
	 */

6298
	for_each_possible_cpu(i) {
E
Eric Dumazet 已提交
6299
		struct softnet_data *sd = &per_cpu(softnet_data, i);
L
Linus Torvalds 已提交
6300

C
Changli Gao 已提交
6301
		memset(sd, 0, sizeof(*sd));
E
Eric Dumazet 已提交
6302
		skb_queue_head_init(&sd->input_pkt_queue);
6303
		skb_queue_head_init(&sd->process_queue);
E
Eric Dumazet 已提交
6304 6305
		sd->completion_queue = NULL;
		INIT_LIST_HEAD(&sd->poll_list);
6306 6307
		sd->output_queue = NULL;
		sd->output_queue_tailp = &sd->output_queue;
E
Eric Dumazet 已提交
6308
#ifdef CONFIG_RPS
E
Eric Dumazet 已提交
6309 6310 6311 6312
		sd->csd.func = rps_trigger_softirq;
		sd->csd.info = sd;
		sd->csd.flags = 0;
		sd->cpu = i;
6313
#endif
T
Tom Herbert 已提交
6314

E
Eric Dumazet 已提交
6315 6316 6317 6318
		sd->backlog.poll = process_backlog;
		sd->backlog.weight = weight_p;
		sd->backlog.gro_list = NULL;
		sd->backlog.gro_count = 0;
6319 6320 6321 6322

#ifdef CONFIG_NET_FLOW_LIMIT
		sd->flow_limit = NULL;
#endif
L
Linus Torvalds 已提交
6323 6324 6325 6326
	}

	dev_boot_phase = 0;

6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341
	/* The loopback device is special if any other network devices
	 * is present in a network namespace the loopback device must
	 * be present. Since we now dynamically allocate and free the
	 * loopback device ensure this invariant is maintained by
	 * keeping the loopback device as the first device on the
	 * list of network devices.  Ensuring the loopback devices
	 * is the first device that appears and the last network device
	 * that disappears.
	 */
	if (register_pernet_device(&loopback_net_ops))
		goto out;

	if (register_pernet_device(&default_device_ops))
		goto out;

6342 6343
	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
L
Linus Torvalds 已提交
6344 6345 6346 6347 6348 6349 6350 6351 6352

	hotcpu_notifier(dev_cpu_callback, 0);
	dst_init();
	rc = 0;
out:
	return rc;
}

subsys_initcall(net_dev_init);