skbuff.c 156.8 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
L
Linus Torvalds 已提交
2 3 4
/*
 *	Routines having to do with the 'struct sk_buff' memory handlers.
 *
5
 *	Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
 *			Florian La Roche <rzsfl@rz.uni-sb.de>
 *
 *	Fixes:
 *		Alan Cox	:	Fixed the worst of the load
 *					balancer bugs.
 *		Dave Platt	:	Interrupt stacking fix.
 *	Richard Kooijman	:	Timestamp fixes.
 *		Alan Cox	:	Changed buffer format.
 *		Alan Cox	:	destructor hook for AF_UNIX etc.
 *		Linus Torvalds	:	Better skb_clone.
 *		Alan Cox	:	Added skb_copy.
 *		Alan Cox	:	Added all the changed routines Linus
 *					only put in the headers
 *		Ray VanTassle	:	Fixed --skb->lock in free
 *		Alan Cox	:	skb_copy copy arp field
 *		Andi Kleen	:	slabified it.
 *		Robert Olsson	:	Removed skb_head_pool
 *
 *	NOTE:
 *		The __skb_ routines should be called with interrupts
 *	disabled, or you better be *real* sure that the operation is atomic
 *	with respect to whatever list is being frobbed (e.g. via lock_sock()
 *	or via disabling bottom half handlers, etc).
 */

/*
 *	The functions in this file will not compile correctly with gcc 2.4.x
 */

J
Joe Perches 已提交
35 36
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

L
Linus Torvalds 已提交
37 38 39 40 41 42 43 44
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/slab.h>
45 46
#include <linux/tcp.h>
#include <linux/udp.h>
47
#include <linux/sctp.h>
L
Linus Torvalds 已提交
48 49 50 51 52 53
#include <linux/netdevice.h>
#ifdef CONFIG_NET_CLS_ACT
#include <net/pkt_sched.h>
#endif
#include <linux/string.h>
#include <linux/skbuff.h>
54
#include <linux/splice.h>
L
Linus Torvalds 已提交
55 56 57
#include <linux/cache.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
58
#include <linux/scatterlist.h>
59
#include <linux/errqueue.h>
60
#include <linux/prefetch.h>
61
#include <linux/if_vlan.h>
62
#include <linux/mpls.h>
L
Linus Torvalds 已提交
63 64 65 66 67

#include <net/protocol.h>
#include <net/dst.h>
#include <net/sock.h>
#include <net/checksum.h>
68
#include <net/ip6_checksum.h>
L
Linus Torvalds 已提交
69
#include <net/xfrm.h>
70
#include <net/mpls.h>
71
#include <net/mptcp.h>
L
Linus Torvalds 已提交
72

73
#include <linux/uaccess.h>
74
#include <trace/events/skb.h>
75
#include <linux/highmem.h>
76 77
#include <linux/capability.h>
#include <linux/user_namespace.h>
78
#include <linux/indirect_call_wrapper.h>
79

80 81
#include "datagram.h"

82 83
struct kmem_cache *skbuff_head_cache __ro_after_init;
static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
84 85 86
#ifdef CONFIG_SKB_EXTENSIONS
static struct kmem_cache *skbuff_ext_cache __ro_after_init;
#endif
87 88
int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
EXPORT_SYMBOL(sysctl_max_skb_frags);
L
Linus Torvalds 已提交
89 90

/**
91 92 93 94
 *	skb_panic - private function for out-of-line support
 *	@skb:	buffer
 *	@sz:	size
 *	@addr:	address
95
 *	@msg:	skb_over_panic or skb_under_panic
L
Linus Torvalds 已提交
96
 *
97 98 99 100
 *	Out-of-line support for skb_put() and skb_push().
 *	Called via the wrapper skb_over_panic() or skb_under_panic().
 *	Keep out of line to prevent kernel bloat.
 *	__builtin_return_address is not used because it is not always reliable.
L
Linus Torvalds 已提交
101
 */
102
static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
103
		      const char msg[])
L
Linus Torvalds 已提交
104
{
105
	pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
106
		 msg, addr, skb->len, sz, skb->head, skb->data,
J
Joe Perches 已提交
107 108
		 (unsigned long)skb->tail, (unsigned long)skb->end,
		 skb->dev ? skb->dev->name : "<NULL>");
L
Linus Torvalds 已提交
109 110 111
	BUG();
}

112
static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
L
Linus Torvalds 已提交
113
{
114
	skb_panic(skb, sz, addr, __func__);
L
Linus Torvalds 已提交
115 116
}

117 118 119 120
static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
{
	skb_panic(skb, sz, addr, __func__);
}
121 122 123 124 125 126 127 128 129 130

/*
 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
 * the caller if emergency pfmemalloc reserves are being used. If it is and
 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
 * may be used. Otherwise, the packet data may be discarded until enough
 * memory is free
 */
#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
	 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
131 132 133

static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
			       unsigned long ip, bool *pfmemalloc)
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
{
	void *obj;
	bool ret_pfmemalloc = false;

	/*
	 * Try a regular allocation, when that fails and we're not entitled
	 * to the reserves, fail.
	 */
	obj = kmalloc_node_track_caller(size,
					flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
					node);
	if (obj || !(gfp_pfmemalloc_allowed(flags)))
		goto out;

	/* Try again but now we are using pfmemalloc reserves */
	ret_pfmemalloc = true;
	obj = kmalloc_node_track_caller(size, flags, node);

out:
	if (pfmemalloc)
		*pfmemalloc = ret_pfmemalloc;

	return obj;
}

L
Linus Torvalds 已提交
159 160 161 162 163 164 165
/* 	Allocate a new skbuff. We do this ourselves so we can fill in a few
 *	'private' fields and also do memory statistics to find all the
 *	[BEEP] leaks.
 *
 */

/**
166
 *	__alloc_skb	-	allocate a network buffer
L
Linus Torvalds 已提交
167 168
 *	@size: size to allocate
 *	@gfp_mask: allocation mask
169 170 171 172
 *	@flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
 *		instead of head cache and allocate a cloned (child) skb.
 *		If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
 *		allocations in case the data is required for writeback
173
 *	@node: numa node to allocate memory on
L
Linus Torvalds 已提交
174 175
 *
 *	Allocate a new &sk_buff. The returned buffer has no headroom and a
176 177
 *	tail room of at least size bytes. The object has a reference count
 *	of one. The return is the buffer. On a failure the return is %NULL.
L
Linus Torvalds 已提交
178 179 180 181
 *
 *	Buffers may only be allocated from interrupts using a @gfp_mask of
 *	%GFP_ATOMIC.
 */
182
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
183
			    int flags, int node)
L
Linus Torvalds 已提交
184
{
185
	struct kmem_cache *cache;
186
	struct skb_shared_info *shinfo;
L
Linus Torvalds 已提交
187 188
	struct sk_buff *skb;
	u8 *data;
189
	bool pfmemalloc;
L
Linus Torvalds 已提交
190

191 192 193 194 195
	cache = (flags & SKB_ALLOC_FCLONE)
		? skbuff_fclone_cache : skbuff_head_cache;

	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
		gfp_mask |= __GFP_MEMALLOC;
196

L
Linus Torvalds 已提交
197
	/* Get the HEAD */
198
	skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
L
Linus Torvalds 已提交
199 200
	if (!skb)
		goto out;
E
Eric Dumazet 已提交
201
	prefetchw(skb);
L
Linus Torvalds 已提交
202

203 204 205 206 207
	/* We do our best to align skb_shared_info on a separate cache
	 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
	 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
	 * Both skb->head and skb_shared_info are cache line aligned.
	 */
208
	size = SKB_DATA_ALIGN(size);
209
	size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
210
	data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
L
Linus Torvalds 已提交
211 212
	if (!data)
		goto nodata;
213 214 215 216 217
	/* kmalloc(size) might give us more room than requested.
	 * Put skb_shared_info exactly at the end of allocated zone,
	 * to allow max possible filling before reallocation.
	 */
	size = SKB_WITH_OVERHEAD(ksize(data));
E
Eric Dumazet 已提交
218
	prefetchw(data + size);
L
Linus Torvalds 已提交
219

220
	/*
221 222 223
	 * Only clear those fields we need to clear, not those that we will
	 * actually initialise below. Hence, don't put any more fields after
	 * the tail pointer in struct sk_buff!
224 225
	 */
	memset(skb, 0, offsetof(struct sk_buff, tail));
226 227
	/* Account for allocated memory : skb + skb->head */
	skb->truesize = SKB_TRUESIZE(size);
228
	skb->pfmemalloc = pfmemalloc;
229
	refcount_set(&skb->users, 1);
L
Linus Torvalds 已提交
230 231
	skb->head = data;
	skb->data = data;
232
	skb_reset_tail_pointer(skb);
233
	skb->end = skb->tail + size;
234 235
	skb->mac_header = (typeof(skb->mac_header))~0U;
	skb->transport_header = (typeof(skb->transport_header))~0U;
236

237 238
	/* make sure we initialize shinfo sequentially */
	shinfo = skb_shinfo(skb);
E
Eric Dumazet 已提交
239
	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
240 241
	atomic_set(&shinfo->dataref, 1);

242
	if (flags & SKB_ALLOC_FCLONE) {
243
		struct sk_buff_fclones *fclones;
L
Linus Torvalds 已提交
244

245 246
		fclones = container_of(skb, struct sk_buff_fclones, skb1);

247
		skb->fclone = SKB_FCLONE_ORIG;
248
		refcount_set(&fclones->fclone_ref, 1);
249

250
		fclones->skb2.fclone = SKB_FCLONE_CLONE;
251
	}
252 253 254

	skb_set_kcov_handle(skb, kcov_common_handle());

L
Linus Torvalds 已提交
255 256 257
out:
	return skb;
nodata:
258
	kmem_cache_free(cache, skb);
L
Linus Torvalds 已提交
259 260 261
	skb = NULL;
	goto out;
}
262
EXPORT_SYMBOL(__alloc_skb);
L
Linus Torvalds 已提交
263

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
/* Caller must provide SKB that is memset cleared */
static struct sk_buff *__build_skb_around(struct sk_buff *skb,
					  void *data, unsigned int frag_size)
{
	struct skb_shared_info *shinfo;
	unsigned int size = frag_size ? : ksize(data);

	size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

	/* Assumes caller memset cleared SKB */
	skb->truesize = SKB_TRUESIZE(size);
	refcount_set(&skb->users, 1);
	skb->head = data;
	skb->data = data;
	skb_reset_tail_pointer(skb);
	skb->end = skb->tail + size;
	skb->mac_header = (typeof(skb->mac_header))~0U;
	skb->transport_header = (typeof(skb->transport_header))~0U;

	/* make sure we initialize shinfo sequentially */
	shinfo = skb_shinfo(skb);
	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
	atomic_set(&shinfo->dataref, 1);

288 289
	skb_set_kcov_handle(skb, kcov_common_handle());

290 291 292
	return skb;
}

E
Eric Dumazet 已提交
293
/**
294
 * __build_skb - build a network buffer
E
Eric Dumazet 已提交
295
 * @data: data buffer provided by caller
296
 * @frag_size: size of data, or 0 if head was kmalloced
E
Eric Dumazet 已提交
297 298
 *
 * Allocate a new &sk_buff. Caller provides space holding head and
299
 * skb_shared_info. @data must have been allocated by kmalloc() only if
300 301
 * @frag_size is 0, otherwise data should come from the page allocator
 *  or vmalloc()
E
Eric Dumazet 已提交
302 303 304 305 306 307 308 309 310 311
 * The return is the new skb buffer.
 * On a failure the return is %NULL, and @data is not freed.
 * Notes :
 *  Before IO, driver allocates only data buffer where NIC put incoming frame
 *  Driver should add room at head (NET_SKB_PAD) and
 *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
 *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
 *  before giving packet to stack.
 *  RX rings only contains data buffers, not full skbs.
 */
312
struct sk_buff *__build_skb(void *data, unsigned int frag_size)
E
Eric Dumazet 已提交
313 314 315 316
{
	struct sk_buff *skb;

	skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
317
	if (unlikely(!skb))
E
Eric Dumazet 已提交
318 319 320 321
		return NULL;

	memset(skb, 0, offsetof(struct sk_buff, tail));

322
	return __build_skb_around(skb, data, frag_size);
E
Eric Dumazet 已提交
323
}
324 325 326 327 328 329 330 331 332 333 334 335

/* build_skb() is wrapper over __build_skb(), that specifically
 * takes care of skb->head and skb->pfmemalloc
 * This means that if @frag_size is not zero, then @data must be backed
 * by a page fragment, not kmalloc() or vmalloc()
 */
struct sk_buff *build_skb(void *data, unsigned int frag_size)
{
	struct sk_buff *skb = __build_skb(data, frag_size);

	if (skb && frag_size) {
		skb->head_frag = 1;
336
		if (page_is_pfmemalloc(virt_to_head_page(data)))
337 338 339 340
			skb->pfmemalloc = 1;
	}
	return skb;
}
E
Eric Dumazet 已提交
341 342
EXPORT_SYMBOL(build_skb);

343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
/**
 * build_skb_around - build a network buffer around provided skb
 * @skb: sk_buff provide by caller, must be memset cleared
 * @data: data buffer provided by caller
 * @frag_size: size of data, or 0 if head was kmalloced
 */
struct sk_buff *build_skb_around(struct sk_buff *skb,
				 void *data, unsigned int frag_size)
{
	if (unlikely(!skb))
		return NULL;

	skb = __build_skb_around(skb, data, frag_size);

	if (skb && frag_size) {
		skb->head_frag = 1;
		if (page_is_pfmemalloc(virt_to_head_page(data)))
			skb->pfmemalloc = 1;
	}
	return skb;
}
EXPORT_SYMBOL(build_skb_around);

366 367 368 369
#define NAPI_SKB_CACHE_SIZE	64

struct napi_alloc_cache {
	struct page_frag_cache page;
370
	unsigned int skb_count;
371 372 373
	void *skb_cache[NAPI_SKB_CACHE_SIZE];
};

374
static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
375
static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
376

377
static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
378
{
379
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
380

381 382 383 384 385 386 387 388
	return page_frag_alloc(&nc->page, fragsz, gfp_mask);
}

void *napi_alloc_frag(unsigned int fragsz)
{
	fragsz = SKB_DATA_ALIGN(fragsz);

	return __napi_alloc_frag(fragsz, GFP_ATOMIC);
389
}
390
EXPORT_SYMBOL(napi_alloc_frag);
391 392 393 394 395 396 397 398 399 400

/**
 * netdev_alloc_frag - allocate a page fragment
 * @fragsz: fragment size
 *
 * Allocates a frag from a page for receive buffer.
 * Uses GFP_ATOMIC allocations.
 */
void *netdev_alloc_frag(unsigned int fragsz)
{
401 402
	struct page_frag_cache *nc;
	void *data;
403

404
	fragsz = SKB_DATA_ALIGN(fragsz);
405 406 407 408 409 410 411 412 413
	if (in_irq() || irqs_disabled()) {
		nc = this_cpu_ptr(&netdev_alloc_cache);
		data = page_frag_alloc(nc, fragsz, GFP_ATOMIC);
	} else {
		local_bh_disable();
		data = __napi_alloc_frag(fragsz, GFP_ATOMIC);
		local_bh_enable();
	}
	return data;
414
}
415
EXPORT_SYMBOL(netdev_alloc_frag);
416

417 418 419
/**
 *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device
 *	@dev: network device to receive on
420
 *	@len: length to allocate
421 422 423 424 425 426 427 428 429
 *	@gfp_mask: get_free_pages mask, passed to alloc_skb
 *
 *	Allocate a new &sk_buff and assign it a usage count of one. The
 *	buffer has NET_SKB_PAD headroom built in. Users should allocate
 *	the headroom they think they need without accounting for the
 *	built in space. The built in space is used for optimisations.
 *
 *	%NULL is returned if there is no free memory.
 */
430 431
struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
				   gfp_t gfp_mask)
432
{
433
	struct page_frag_cache *nc;
434
	struct sk_buff *skb;
435 436 437 438
	bool pfmemalloc;
	void *data;

	len += NET_SKB_PAD;
439

440
	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
441
	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
442 443 444 445 446
		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
		if (!skb)
			goto skb_fail;
		goto skb_success;
	}
447

448 449 450 451 452 453
	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	len = SKB_DATA_ALIGN(len);

	if (sk_memalloc_socks())
		gfp_mask |= __GFP_MEMALLOC;

454 455 456 457 458 459 460 461 462 463 464
	if (in_irq() || irqs_disabled()) {
		nc = this_cpu_ptr(&netdev_alloc_cache);
		data = page_frag_alloc(nc, len, gfp_mask);
		pfmemalloc = nc->pfmemalloc;
	} else {
		local_bh_disable();
		nc = this_cpu_ptr(&napi_alloc_cache.page);
		data = page_frag_alloc(nc, len, gfp_mask);
		pfmemalloc = nc->pfmemalloc;
		local_bh_enable();
	}
465 466 467 468 469 470

	if (unlikely(!data))
		return NULL;

	skb = __build_skb(data, len);
	if (unlikely(!skb)) {
471
		skb_free_frag(data);
472
		return NULL;
473
	}
474

475 476 477 478
	if (pfmemalloc)
		skb->pfmemalloc = 1;
	skb->head_frag = 1;

479
skb_success:
480 481 482
	skb_reserve(skb, NET_SKB_PAD);
	skb->dev = dev;

483
skb_fail:
484 485
	return skb;
}
486
EXPORT_SYMBOL(__netdev_alloc_skb);
L
Linus Torvalds 已提交
487

488 489 490
/**
 *	__napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
 *	@napi: napi instance this buffer was allocated for
491
 *	@len: length to allocate
492 493 494 495 496 497 498 499 500
 *	@gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
 *
 *	Allocate a new sk_buff for use in NAPI receive.  This buffer will
 *	attempt to allocate the head from a special reserved region used
 *	only for NAPI Rx allocation.  By doing this we can save several
 *	CPU cycles by avoiding having to disable and re-enable IRQs.
 *
 *	%NULL is returned if there is no free memory.
 */
501 502
struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
				 gfp_t gfp_mask)
503
{
504
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
505
	struct sk_buff *skb;
506 507 508
	void *data;

	len += NET_SKB_PAD + NET_IP_ALIGN;
509

510
	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
511
	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
512 513 514 515 516
		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
		if (!skb)
			goto skb_fail;
		goto skb_success;
	}
517 518 519 520 521 522

	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	len = SKB_DATA_ALIGN(len);

	if (sk_memalloc_socks())
		gfp_mask |= __GFP_MEMALLOC;
523

524
	data = page_frag_alloc(&nc->page, len, gfp_mask);
525 526 527 528 529
	if (unlikely(!data))
		return NULL;

	skb = __build_skb(data, len);
	if (unlikely(!skb)) {
530
		skb_free_frag(data);
531
		return NULL;
532 533
	}

534
	if (nc->page.pfmemalloc)
535 536 537
		skb->pfmemalloc = 1;
	skb->head_frag = 1;

538
skb_success:
539 540 541
	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
	skb->dev = napi->dev;

542
skb_fail:
543 544 545 546
	return skb;
}
EXPORT_SYMBOL(__napi_alloc_skb);

547
void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
548
		     int size, unsigned int truesize)
549 550 551 552
{
	skb_fill_page_desc(skb, i, page, off, size);
	skb->len += size;
	skb->data_len += size;
553
	skb->truesize += truesize;
554 555 556
}
EXPORT_SYMBOL(skb_add_rx_frag);

557 558 559 560 561 562 563 564 565 566 567 568
void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
			  unsigned int truesize)
{
	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

	skb_frag_size_add(frag, size);
	skb->len += size;
	skb->data_len += size;
	skb->truesize += truesize;
}
EXPORT_SYMBOL(skb_coalesce_rx_frag);

569
static void skb_drop_list(struct sk_buff **listp)
L
Linus Torvalds 已提交
570
{
571
	kfree_skb_list(*listp);
572
	*listp = NULL;
L
Linus Torvalds 已提交
573 574
}

575 576 577 578 579
static inline void skb_drop_fraglist(struct sk_buff *skb)
{
	skb_drop_list(&skb_shinfo(skb)->frag_list);
}

L
Linus Torvalds 已提交
580 581 582 583
static void skb_clone_fraglist(struct sk_buff *skb)
{
	struct sk_buff *list;

584
	skb_walk_frags(skb, list)
L
Linus Torvalds 已提交
585 586 587
		skb_get(list);
}

588 589
static void skb_free_head(struct sk_buff *skb)
{
590 591
	unsigned char *head = skb->head;

592
	if (skb->head_frag)
593
		skb_free_frag(head);
594
	else
595
		kfree(head);
596 597
}

598
static void skb_release_data(struct sk_buff *skb)
L
Linus Torvalds 已提交
599
{
600 601
	struct skb_shared_info *shinfo = skb_shinfo(skb);
	int i;
L
Linus Torvalds 已提交
602

603 604 605 606
	if (skb->cloned &&
	    atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
			      &shinfo->dataref))
		return;
607

608 609
	for (i = 0; i < shinfo->nr_frags; i++)
		__skb_frag_unref(&shinfo->frags[i]);
610

611 612 613
	if (shinfo->frag_list)
		kfree_skb_list(shinfo->frag_list);

614
	skb_zcopy_clear(skb, true);
615
	skb_free_head(skb);
L
Linus Torvalds 已提交
616 617 618 619 620
}

/*
 *	Free an skbuff by memory without cleaning the state.
 */
621
static void kfree_skbmem(struct sk_buff *skb)
L
Linus Torvalds 已提交
622
{
623
	struct sk_buff_fclones *fclones;
624 625 626 627

	switch (skb->fclone) {
	case SKB_FCLONE_UNAVAILABLE:
		kmem_cache_free(skbuff_head_cache, skb);
628
		return;
629 630

	case SKB_FCLONE_ORIG:
631
		fclones = container_of(skb, struct sk_buff_fclones, skb1);
632

633 634 635
		/* We usually free the clone (TX completion) before original skb
		 * This test would have no chance to be true for the clone,
		 * while here, branch prediction will be good.
636
		 */
637
		if (refcount_read(&fclones->fclone_ref) == 1)
638 639
			goto fastpath;
		break;
640

641 642
	default: /* SKB_FCLONE_CLONE */
		fclones = container_of(skb, struct sk_buff_fclones, skb2);
643
		break;
644
	}
645
	if (!refcount_dec_and_test(&fclones->fclone_ref))
646 647 648
		return;
fastpath:
	kmem_cache_free(skbuff_fclone_cache, fclones);
L
Linus Torvalds 已提交
649 650
}

651
void skb_release_head_state(struct sk_buff *skb)
L
Linus Torvalds 已提交
652
{
E
Eric Dumazet 已提交
653
	skb_dst_drop(skb);
654 655
	if (skb->destructor) {
		WARN_ON(in_irq());
L
Linus Torvalds 已提交
656 657
		skb->destructor(skb);
	}
I
Igor Maravić 已提交
658
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
659
	nf_conntrack_put(skb_nfct(skb));
L
Linus Torvalds 已提交
660
#endif
661
	skb_ext_put(skb);
662 663 664 665 666 667
}

/* Free everything but the sk_buff shell. */
static void skb_release_all(struct sk_buff *skb)
{
	skb_release_head_state(skb);
668 669
	if (likely(skb->head))
		skb_release_data(skb);
670 671 672 673 674 675 676 677 678 679
}

/**
 *	__kfree_skb - private function
 *	@skb: buffer
 *
 *	Free an sk_buff. Release anything attached to the buffer.
 *	Clean the state. This is an internal helper function. Users should
 *	always call kfree_skb
 */
L
Linus Torvalds 已提交
680

681 682 683
void __kfree_skb(struct sk_buff *skb)
{
	skb_release_all(skb);
L
Linus Torvalds 已提交
684 685
	kfree_skbmem(skb);
}
686
EXPORT_SYMBOL(__kfree_skb);
L
Linus Torvalds 已提交
687

688 689 690 691 692 693 694 695 696
/**
 *	kfree_skb - free an sk_buff
 *	@skb: buffer to free
 *
 *	Drop a reference to the buffer and free it if the usage count has
 *	hit zero.
 */
void kfree_skb(struct sk_buff *skb)
{
697
	if (!skb_unref(skb))
698
		return;
699

700
	trace_kfree_skb(skb, __builtin_return_address(0));
701 702
	__kfree_skb(skb);
}
703
EXPORT_SYMBOL(kfree_skb);
704

705 706 707 708 709 710 711 712 713 714 715
void kfree_skb_list(struct sk_buff *segs)
{
	while (segs) {
		struct sk_buff *next = segs->next;

		kfree_skb(segs);
		segs = next;
	}
}
EXPORT_SYMBOL(kfree_skb_list);

716 717 718 719
/* Dump skb information and contents.
 *
 * Must only be called from net_ratelimit()-ed paths.
 *
720
 * Dumps whole packets if full_pkt, only headers otherwise.
721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764
 */
void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
{
	struct skb_shared_info *sh = skb_shinfo(skb);
	struct net_device *dev = skb->dev;
	struct sock *sk = skb->sk;
	struct sk_buff *list_skb;
	bool has_mac, has_trans;
	int headroom, tailroom;
	int i, len, seg_len;

	if (full_pkt)
		len = skb->len;
	else
		len = min_t(int, skb->len, MAX_HEADER + 128);

	headroom = skb_headroom(skb);
	tailroom = skb_tailroom(skb);

	has_mac = skb_mac_header_was_set(skb);
	has_trans = skb_transport_header_was_set(skb);

	printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
	       "mac=(%d,%d) net=(%d,%d) trans=%d\n"
	       "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
	       "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
	       "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
	       level, skb->len, headroom, skb_headlen(skb), tailroom,
	       has_mac ? skb->mac_header : -1,
	       has_mac ? skb_mac_header_len(skb) : -1,
	       skb->network_header,
	       has_trans ? skb_network_header_len(skb) : -1,
	       has_trans ? skb->transport_header : -1,
	       sh->tx_flags, sh->nr_frags,
	       sh->gso_size, sh->gso_type, sh->gso_segs,
	       skb->csum, skb->ip_summed, skb->csum_complete_sw,
	       skb->csum_valid, skb->csum_level,
	       skb->hash, skb->sw_hash, skb->l4_hash,
	       ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);

	if (dev)
		printk("%sdev name=%s feat=0x%pNF\n",
		       level, dev->name, &dev->features);
	if (sk)
765
		printk("%ssk family=%hu type=%u proto=%u\n",
766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787
		       level, sk->sk_family, sk->sk_type, sk->sk_protocol);

	if (full_pkt && headroom)
		print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
			       16, 1, skb->head, headroom, false);

	seg_len = min_t(int, skb_headlen(skb), len);
	if (seg_len)
		print_hex_dump(level, "skb linear:   ", DUMP_PREFIX_OFFSET,
			       16, 1, skb->data, seg_len, false);
	len -= seg_len;

	if (full_pkt && tailroom)
		print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
			       16, 1, skb_tail_pointer(skb), tailroom, false);

	for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
		u32 p_off, p_len, copied;
		struct page *p;
		u8 *vaddr;

788
		skb_frag_foreach_page(frag, skb_frag_off(frag),
789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810
				      skb_frag_size(frag), p, p_off, p_len,
				      copied) {
			seg_len = min_t(int, p_len, len);
			vaddr = kmap_atomic(p);
			print_hex_dump(level, "skb frag:     ",
				       DUMP_PREFIX_OFFSET,
				       16, 1, vaddr + p_off, seg_len, false);
			kunmap_atomic(vaddr);
			len -= seg_len;
			if (!len)
				break;
		}
	}

	if (full_pkt && skb_has_frag_list(skb)) {
		printk("skb fraglist:\n");
		skb_walk_frags(skb, list_skb)
			skb_dump(level, list_skb, true);
	}
}
EXPORT_SYMBOL(skb_dump);

811 812 813 814 815 816 817 818 819
/**
 *	skb_tx_error - report an sk_buff xmit error
 *	@skb: buffer that triggered an error
 *
 *	Report xmit error if a device callback is tracking this skb.
 *	skb must be freed afterwards.
 */
void skb_tx_error(struct sk_buff *skb)
{
820
	skb_zcopy_clear(skb, true);
821 822 823
}
EXPORT_SYMBOL(skb_tx_error);

824
#ifdef CONFIG_TRACEPOINTS
825 826 827 828 829 830 831 832 833 834
/**
 *	consume_skb - free an skbuff
 *	@skb: buffer to free
 *
 *	Drop a ref to the buffer and free it if the usage count has hit zero
 *	Functions identically to kfree_skb, but kfree_skb assumes that the frame
 *	is being dropped after a failure and notes that
 */
void consume_skb(struct sk_buff *skb)
{
835
	if (!skb_unref(skb))
836
		return;
837

838
	trace_consume_skb(skb);
839 840 841
	__kfree_skb(skb);
}
EXPORT_SYMBOL(consume_skb);
842
#endif
843

844
/**
845
 *	__consume_stateless_skb - free an skbuff, assuming it is stateless
846 847
 *	@skb: buffer to free
 *
848 849
 *	Alike consume_skb(), but this variant assumes that this is the last
 *	skb reference and all the head states have been already dropped
850
 */
851
void __consume_stateless_skb(struct sk_buff *skb)
852 853
{
	trace_consume_skb(skb);
854
	skb_release_data(skb);
855 856 857
	kfree_skbmem(skb);
}

858 859 860 861 862 863 864 865 866 867 868 869
void __kfree_skb_flush(void)
{
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);

	/* flush skb_cache if containing objects */
	if (nc->skb_count) {
		kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
				     nc->skb_cache);
		nc->skb_count = 0;
	}
}

870
static inline void _kfree_skb_defer(struct sk_buff *skb)
871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891
{
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);

	/* drop skb->head and call any destructors for packet */
	skb_release_all(skb);

	/* record skb to CPU local list */
	nc->skb_cache[nc->skb_count++] = skb;

#ifdef CONFIG_SLUB
	/* SLUB writes into objects when freeing */
	prefetchw(skb);
#endif

	/* flush skb_cache if it is filled */
	if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
		kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
				     nc->skb_cache);
		nc->skb_count = 0;
	}
}
892 893 894 895
void __kfree_skb_defer(struct sk_buff *skb)
{
	_kfree_skb_defer(skb);
}
896 897 898

void napi_consume_skb(struct sk_buff *skb, int budget)
{
899
	/* Zero budget indicate non-NAPI context called us, like netpoll */
900
	if (unlikely(!budget)) {
901
		dev_consume_skb_any(skb);
902 903 904
		return;
	}

905 906
	lockdep_assert_in_softirq();

907
	if (!skb_unref(skb))
908
		return;
909

910 911 912 913
	/* if reaching here SKB is ready to free */
	trace_consume_skb(skb);

	/* if SKB is a clone, don't handle this case */
914
	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
915 916 917 918
		__kfree_skb(skb);
		return;
	}

919
	_kfree_skb_defer(skb);
920 921 922
}
EXPORT_SYMBOL(napi_consume_skb);

923 924 925 926 927 928 929
/* Make sure a field is enclosed inside headers_start/headers_end section */
#define CHECK_SKB_FIELD(field) \
	BUILD_BUG_ON(offsetof(struct sk_buff, field) <		\
		     offsetof(struct sk_buff, headers_start));	\
	BUILD_BUG_ON(offsetof(struct sk_buff, field) >		\
		     offsetof(struct sk_buff, headers_end));	\

930 931 932
static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
{
	new->tstamp		= old->tstamp;
933
	/* We do not copy old->sk */
934
	new->dev		= old->dev;
935
	memcpy(new->cb, old->cb, sizeof(old->cb));
936
	skb_dst_copy(new, old);
937
	__skb_ext_copy(new, old);
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
	__nf_copy(new, old, false);

	/* Note : this field could be in headers_start/headers_end section
	 * It is not yet because we do not want to have a 16 bit hole
	 */
	new->queue_mapping = old->queue_mapping;

	memcpy(&new->headers_start, &old->headers_start,
	       offsetof(struct sk_buff, headers_end) -
	       offsetof(struct sk_buff, headers_start));
	CHECK_SKB_FIELD(protocol);
	CHECK_SKB_FIELD(csum);
	CHECK_SKB_FIELD(hash);
	CHECK_SKB_FIELD(priority);
	CHECK_SKB_FIELD(skb_iif);
	CHECK_SKB_FIELD(vlan_proto);
	CHECK_SKB_FIELD(vlan_tci);
	CHECK_SKB_FIELD(transport_header);
	CHECK_SKB_FIELD(network_header);
	CHECK_SKB_FIELD(mac_header);
	CHECK_SKB_FIELD(inner_protocol);
	CHECK_SKB_FIELD(inner_transport_header);
	CHECK_SKB_FIELD(inner_network_header);
	CHECK_SKB_FIELD(inner_mac_header);
	CHECK_SKB_FIELD(mark);
#ifdef CONFIG_NETWORK_SECMARK
	CHECK_SKB_FIELD(secmark);
#endif
#ifdef CONFIG_NET_RX_BUSY_POLL
	CHECK_SKB_FIELD(napi_id);
968
#endif
969 970 971
#ifdef CONFIG_XPS
	CHECK_SKB_FIELD(sender_cpu);
#endif
972
#ifdef CONFIG_NET_SCHED
973
	CHECK_SKB_FIELD(tc_index);
974
#endif
975

976 977
}

978 979 980 981
/*
 * You should not add any new code to this function.  Add it to
 * __copy_skb_header above instead.
 */
H
Herbert Xu 已提交
982
static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
L
Linus Torvalds 已提交
983 984 985 986 987
{
#define C(x) n->x = skb->x

	n->next = n->prev = NULL;
	n->sk = NULL;
988 989
	__copy_skb_header(n, skb);

L
Linus Torvalds 已提交
990 991
	C(len);
	C(data_len);
992
	C(mac_len);
993
	n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
994
	n->cloned = 1;
L
Linus Torvalds 已提交
995
	n->nohdr = 0;
996
	n->peeked = 0;
997
	C(pfmemalloc);
L
Linus Torvalds 已提交
998 999 1000
	n->destructor = NULL;
	C(tail);
	C(end);
1001
	C(head);
1002
	C(head_frag);
1003 1004
	C(data);
	C(truesize);
1005
	refcount_set(&n->users, 1);
L
Linus Torvalds 已提交
1006 1007 1008 1009 1010

	atomic_inc(&(skb_shinfo(skb)->dataref));
	skb->cloned = 1;

	return n;
H
Herbert Xu 已提交
1011 1012 1013
#undef C
}

1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
/**
 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
 * @first: first sk_buff of the msg
 */
struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
{
	struct sk_buff *n;

	n = alloc_skb(0, GFP_ATOMIC);
	if (!n)
		return NULL;

	n->len = first->len;
	n->data_len = first->len;
	n->truesize = first->truesize;

	skb_shinfo(n)->frag_list = first;

	__copy_skb_header(n, first);
	n->destructor = NULL;

	return n;
}
EXPORT_SYMBOL_GPL(alloc_skb_for_msg);

H
Herbert Xu 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
/**
 *	skb_morph	-	morph one skb into another
 *	@dst: the skb to receive the contents
 *	@src: the skb to supply the contents
 *
 *	This is identical to skb_clone except that the target skb is
 *	supplied by the user.
 *
 *	The target skb is returned upon exit.
 */
struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
{
1051
	skb_release_all(dst);
H
Herbert Xu 已提交
1052 1053 1054 1055
	return __skb_clone(dst, src);
}
EXPORT_SYMBOL_GPL(skb_morph);

1056
int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
{
	unsigned long max_pg, num_pg, new_pg, old_pg;
	struct user_struct *user;

	if (capable(CAP_IPC_LOCK) || !size)
		return 0;

	num_pg = (size >> PAGE_SHIFT) + 2;	/* worst case */
	max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	user = mmp->user ? : current_user();

	do {
		old_pg = atomic_long_read(&user->locked_vm);
		new_pg = old_pg + num_pg;
		if (new_pg > max_pg)
			return -ENOBUFS;
	} while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
		 old_pg);

	if (!mmp->user) {
		mmp->user = get_uid(user);
		mmp->num_pg = num_pg;
	} else {
		mmp->num_pg += num_pg;
	}

	return 0;
}
1085
EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
1086

1087
void mm_unaccount_pinned_pages(struct mmpin *mmp)
1088 1089 1090 1091 1092 1093
{
	if (mmp->user) {
		atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
		free_uid(mmp->user);
	}
}
1094
EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
1095

1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
{
	struct ubuf_info *uarg;
	struct sk_buff *skb;

	WARN_ON_ONCE(!in_task());

	skb = sock_omalloc(sk, 0, GFP_KERNEL);
	if (!skb)
		return NULL;

	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
	uarg = (void *)skb->cb;
1109 1110 1111 1112 1113 1114
	uarg->mmp.user = NULL;

	if (mm_account_pinned_pages(&uarg->mmp, size)) {
		kfree_skb(skb);
		return NULL;
	}
1115 1116

	uarg->callback = sock_zerocopy_callback;
1117 1118 1119
	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
	uarg->len = 1;
	uarg->bytelen = size;
1120
	uarg->zerocopy = 1;
1121
	refcount_set(&uarg->refcnt, 1);
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
	sock_hold(sk);

	return uarg;
}
EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);

static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
{
	return container_of((void *)uarg, struct sk_buff, cb);
}

1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
					struct ubuf_info *uarg)
{
	if (uarg) {
		const u32 byte_limit = 1 << 19;		/* limit to a few TSO */
		u32 bytelen, next;

		/* realloc only when socket is locked (TCP, UDP cork),
		 * so uarg->len and sk_zckey access is serialized
		 */
		if (!sock_owned_by_user(sk)) {
			WARN_ON_ONCE(1);
			return NULL;
		}

		bytelen = uarg->bytelen + size;
		if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
			/* TCP can create new skb to attach new uarg */
			if (sk->sk_type == SOCK_STREAM)
				goto new_alloc;
			return NULL;
		}

		next = (u32)atomic_read(&sk->sk_zckey);
		if ((u32)(uarg->id + uarg->len) == next) {
1158 1159
			if (mm_account_pinned_pages(&uarg->mmp, size))
				return NULL;
1160 1161 1162
			uarg->len++;
			uarg->bytelen = bytelen;
			atomic_set(&sk->sk_zckey, ++next);
1163 1164 1165 1166 1167

			/* no extra ref when appending to datagram (MSG_MORE) */
			if (sk->sk_type == SOCK_STREAM)
				sock_zerocopy_get(uarg);

1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
			return uarg;
		}
	}

new_alloc:
	return sock_zerocopy_alloc(sk, size);
}
EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);

static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
{
	struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
	u32 old_lo, old_hi;
	u64 sum_len;

	old_lo = serr->ee.ee_info;
	old_hi = serr->ee.ee_data;
	sum_len = old_hi - old_lo + 1ULL + len;

	if (sum_len >= (1ULL << 32))
		return false;

	if (lo != old_hi + 1)
		return false;

	serr->ee.ee_data += len;
	return true;
}

1197 1198
void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
{
1199
	struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1200 1201
	struct sock_exterr_skb *serr;
	struct sock *sk = skb->sk;
1202 1203 1204 1205
	struct sk_buff_head *q;
	unsigned long flags;
	u32 lo, hi;
	u16 len;
1206

1207 1208
	mm_unaccount_pinned_pages(&uarg->mmp);

1209 1210 1211 1212
	/* if !len, there was only 1 call, and it was aborted
	 * so do not queue a completion notification
	 */
	if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1213 1214
		goto release;

1215 1216 1217 1218
	len = uarg->len;
	lo = uarg->id;
	hi = uarg->id + len - 1;

1219 1220 1221 1222
	serr = SKB_EXT_ERR(skb);
	memset(serr, 0, sizeof(*serr));
	serr->ee.ee_errno = 0;
	serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1223 1224
	serr->ee.ee_data = hi;
	serr->ee.ee_info = lo;
1225 1226 1227
	if (!success)
		serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;

1228 1229 1230 1231 1232 1233 1234 1235 1236
	q = &sk->sk_error_queue;
	spin_lock_irqsave(&q->lock, flags);
	tail = skb_peek_tail(q);
	if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
	    !skb_zerocopy_notify_extend(tail, lo, len)) {
		__skb_queue_tail(q, skb);
		skb = NULL;
	}
	spin_unlock_irqrestore(&q->lock, flags);
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247

	sk->sk_error_report(sk);

release:
	consume_skb(skb);
	sock_put(sk);
}
EXPORT_SYMBOL_GPL(sock_zerocopy_callback);

void sock_zerocopy_put(struct ubuf_info *uarg)
{
1248
	if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
1249 1250 1251 1252 1253 1254 1255 1256
		if (uarg->callback)
			uarg->callback(uarg, uarg->zerocopy);
		else
			consume_skb(skb_from_uarg(uarg));
	}
}
EXPORT_SYMBOL_GPL(sock_zerocopy_put);

1257
void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1258 1259 1260 1261 1262
{
	if (uarg) {
		struct sock *sk = skb_from_uarg(uarg)->sk;

		atomic_dec(&sk->sk_zckey);
1263
		uarg->len--;
1264

1265 1266
		if (have_uref)
			sock_zerocopy_put(uarg);
1267 1268 1269 1270
	}
}
EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);

W
Willem de Bruijn 已提交
1271 1272 1273 1274 1275 1276
int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
{
	return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
}
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);

1277 1278 1279 1280
int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
			     struct msghdr *msg, int len,
			     struct ubuf_info *uarg)
{
1281
	struct ubuf_info *orig_uarg = skb_zcopy(skb);
1282 1283 1284
	struct iov_iter orig_iter = msg->msg_iter;
	int err, orig_len = skb->len;

1285 1286 1287 1288 1289 1290
	/* An skb can only point to one uarg. This edge case happens when
	 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
	 */
	if (orig_uarg && uarg != orig_uarg)
		return -EEXIST;

1291 1292
	err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
	if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1293 1294
		struct sock *save_sk = skb->sk;

1295 1296
		/* Streams do not free skb on error. Reset to prev state. */
		msg->msg_iter = orig_iter;
1297
		skb->sk = sk;
1298
		___pskb_trim(skb, orig_len);
1299
		skb->sk = save_sk;
1300 1301 1302
		return err;
	}

1303
	skb_zcopy_set(skb, uarg, NULL);
1304 1305 1306 1307
	return skb->len - orig_len;
}
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);

1308
static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
			      gfp_t gfp_mask)
{
	if (skb_zcopy(orig)) {
		if (skb_zcopy(nskb)) {
			/* !gfp_mask callers are verified to !skb_zcopy(nskb) */
			if (!gfp_mask) {
				WARN_ON_ONCE(1);
				return -ENOMEM;
			}
			if (skb_uarg(nskb) == skb_uarg(orig))
				return 0;
			if (skb_copy_ubufs(nskb, GFP_ATOMIC))
				return -EIO;
		}
1323
		skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1324 1325 1326 1327
	}
	return 0;
}

1328 1329
/**
 *	skb_copy_ubufs	-	copy userspace skb frags buffers to kernel
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343
 *	@skb: the skb to modify
 *	@gfp_mask: allocation priority
 *
 *	This must be called on SKBTX_DEV_ZEROCOPY skb.
 *	It will copy all frags into kernel and drop the reference
 *	to userspace pages.
 *
 *	If this function is called from an interrupt gfp_mask() must be
 *	%GFP_ATOMIC.
 *
 *	Returns 0 on success or a negative error code on failure
 *	to allocate kernel memory to copy to.
 */
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1344 1345 1346
{
	int num_frags = skb_shinfo(skb)->nr_frags;
	struct page *page, *head = NULL;
1347 1348
	int i, new_frags;
	u32 d_off;
1349

1350 1351
	if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
		return -EINVAL;
1352

1353 1354 1355
	if (!num_frags)
		goto release;

1356 1357
	new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
	for (i = 0; i < new_frags; i++) {
1358
		page = alloc_page(gfp_mask);
1359 1360
		if (!page) {
			while (head) {
1361
				struct page *next = (struct page *)page_private(head);
1362 1363 1364 1365 1366
				put_page(head);
				head = next;
			}
			return -ENOMEM;
		}
1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
		set_page_private(page, (unsigned long)head);
		head = page;
	}

	page = head;
	d_off = 0;
	for (i = 0; i < num_frags; i++) {
		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
		u32 p_off, p_len, copied;
		struct page *p;
		u8 *vaddr;
1378

1379
		skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
1380
				      p, p_off, p_len, copied) {
1381
			u32 copy, done = 0;
1382
			vaddr = kmap_atomic(p);
1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394

			while (done < p_len) {
				if (d_off == PAGE_SIZE) {
					d_off = 0;
					page = (struct page *)page_private(page);
				}
				copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
				memcpy(page_address(page) + d_off,
				       vaddr + p_off + done, copy);
				done += copy;
				d_off += copy;
			}
1395 1396
			kunmap_atomic(vaddr);
		}
1397 1398 1399
	}

	/* skb frags release userspace buffers */
1400
	for (i = 0; i < num_frags; i++)
1401
		skb_frag_unref(skb, i);
1402 1403

	/* skb frags point to kernel buffers */
1404 1405
	for (i = 0; i < new_frags - 1; i++) {
		__skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1406
		head = (struct page *)page_private(head);
1407
	}
1408 1409
	__skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
	skb_shinfo(skb)->nr_frags = new_frags;
1410

1411
release:
1412
	skb_zcopy_clear(skb, false);
1413 1414
	return 0;
}
1415
EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1416

H
Herbert Xu 已提交
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
/**
 *	skb_clone	-	duplicate an sk_buff
 *	@skb: buffer to clone
 *	@gfp_mask: allocation priority
 *
 *	Duplicate an &sk_buff. The new one is not owned by a socket. Both
 *	copies share the same packet data but not structure. The new
 *	buffer has a reference count of 1. If the allocation fails the
 *	function returns %NULL otherwise the new buffer is returned.
 *
 *	If this function is called from an interrupt gfp_mask() must be
 *	%GFP_ATOMIC.
 */

struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
{
1433 1434 1435
	struct sk_buff_fclones *fclones = container_of(skb,
						       struct sk_buff_fclones,
						       skb1);
1436
	struct sk_buff *n;
H
Herbert Xu 已提交
1437

1438 1439
	if (skb_orphan_frags(skb, gfp_mask))
		return NULL;
1440

H
Herbert Xu 已提交
1441
	if (skb->fclone == SKB_FCLONE_ORIG &&
1442
	    refcount_read(&fclones->fclone_ref) == 1) {
1443
		n = &fclones->skb2;
1444
		refcount_set(&fclones->fclone_ref, 2);
H
Herbert Xu 已提交
1445
	} else {
1446 1447 1448
		if (skb_pfmemalloc(skb))
			gfp_mask |= __GFP_MEMALLOC;

H
Herbert Xu 已提交
1449 1450 1451
		n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
		if (!n)
			return NULL;
1452

H
Herbert Xu 已提交
1453 1454 1455 1456
		n->fclone = SKB_FCLONE_UNAVAILABLE;
	}

	return __skb_clone(n, skb);
L
Linus Torvalds 已提交
1457
}
1458
EXPORT_SYMBOL(skb_clone);
L
Linus Torvalds 已提交
1459

1460
void skb_headers_offset_update(struct sk_buff *skb, int off)
1461
{
1462 1463 1464
	/* Only adjust this if it actually is csum_start rather than csum */
	if (skb->ip_summed == CHECKSUM_PARTIAL)
		skb->csum_start += off;
1465 1466 1467 1468 1469 1470 1471
	/* {transport,network,mac}_header and tail are relative to skb->head */
	skb->transport_header += off;
	skb->network_header   += off;
	if (skb_mac_header_was_set(skb))
		skb->mac_header += off;
	skb->inner_transport_header += off;
	skb->inner_network_header += off;
1472
	skb->inner_mac_header += off;
1473
}
1474
EXPORT_SYMBOL(skb_headers_offset_update);
1475

1476
void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
L
Linus Torvalds 已提交
1477
{
1478 1479
	__copy_skb_header(new, old);

1480 1481 1482
	skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
	skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
	skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
L
Linus Torvalds 已提交
1483
}
1484
EXPORT_SYMBOL(skb_copy_header);
L
Linus Torvalds 已提交
1485

1486 1487 1488 1489 1490 1491 1492
static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
{
	if (skb_pfmemalloc(skb))
		return SKB_ALLOC_RX;
	return 0;
}

L
Linus Torvalds 已提交
1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
/**
 *	skb_copy	-	create private copy of an sk_buff
 *	@skb: buffer to copy
 *	@gfp_mask: allocation priority
 *
 *	Make a copy of both an &sk_buff and its data. This is used when the
 *	caller wishes to modify the data and needs a private copy of the
 *	data to alter. Returns %NULL on failure or the pointer to the buffer
 *	on success. The returned buffer has a reference count of 1.
 *
 *	As by-product this function converts non-linear &sk_buff to linear
 *	one, so that &sk_buff becomes completely private and caller is allowed
 *	to modify all the data of returned buffer. This means that this
 *	function is not recommended for use in circumstances when only
 *	header is going to be modified. Use pskb_copy() instead.
 */

1510
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
L
Linus Torvalds 已提交
1511
{
E
Eric Dumazet 已提交
1512
	int headerlen = skb_headroom(skb);
1513
	unsigned int size = skb_end_offset(skb) + skb->data_len;
1514 1515
	struct sk_buff *n = __alloc_skb(size, gfp_mask,
					skb_alloc_rx_flag(skb), NUMA_NO_NODE);
E
Eric Dumazet 已提交
1516

L
Linus Torvalds 已提交
1517 1518 1519 1520 1521 1522 1523 1524
	if (!n)
		return NULL;

	/* Set the data pointer */
	skb_reserve(n, headerlen);
	/* Set the tail pointer and length */
	skb_put(n, skb->len);

1525
	BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
L
Linus Torvalds 已提交
1526

1527
	skb_copy_header(n, skb);
L
Linus Torvalds 已提交
1528 1529
	return n;
}
1530
EXPORT_SYMBOL(skb_copy);
L
Linus Torvalds 已提交
1531 1532

/**
1533
 *	__pskb_copy_fclone	-  create copy of an sk_buff with private head.
L
Linus Torvalds 已提交
1534
 *	@skb: buffer to copy
1535
 *	@headroom: headroom of new skb
L
Linus Torvalds 已提交
1536
 *	@gfp_mask: allocation priority
1537 1538 1539
 *	@fclone: if true allocate the copy of the skb from the fclone
 *	cache instead of the head cache; it is recommended to set this
 *	to true for the cases where the copy will likely be cloned
L
Linus Torvalds 已提交
1540 1541 1542 1543 1544 1545 1546 1547 1548
 *
 *	Make a copy of both an &sk_buff and part of its data, located
 *	in header. Fragmented data remain shared. This is used when
 *	the caller wishes to modify only header of &sk_buff and needs
 *	private copy of the header to alter. Returns %NULL on failure
 *	or the pointer to the buffer on success.
 *	The returned buffer has a reference count of 1.
 */

1549 1550
struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
				   gfp_t gfp_mask, bool fclone)
L
Linus Torvalds 已提交
1551
{
1552
	unsigned int size = skb_headlen(skb) + headroom;
1553 1554
	int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
	struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
E
Eric Dumazet 已提交
1555

L
Linus Torvalds 已提交
1556 1557 1558 1559
	if (!n)
		goto out;

	/* Set the data pointer */
1560
	skb_reserve(n, headroom);
L
Linus Torvalds 已提交
1561 1562 1563
	/* Set the tail pointer and length */
	skb_put(n, skb_headlen(skb));
	/* Copy the bytes */
1564
	skb_copy_from_linear_data(skb, n->data, n->len);
L
Linus Torvalds 已提交
1565

1566
	n->truesize += skb->data_len;
L
Linus Torvalds 已提交
1567 1568 1569 1570 1571 1572
	n->data_len  = skb->data_len;
	n->len	     = skb->len;

	if (skb_shinfo(skb)->nr_frags) {
		int i;

1573 1574
		if (skb_orphan_frags(skb, gfp_mask) ||
		    skb_zerocopy_clone(n, skb, gfp_mask)) {
1575 1576 1577
			kfree_skb(n);
			n = NULL;
			goto out;
1578
		}
L
Linus Torvalds 已提交
1579 1580
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
			skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1581
			skb_frag_ref(skb, i);
L
Linus Torvalds 已提交
1582 1583 1584 1585
		}
		skb_shinfo(n)->nr_frags = i;
	}

1586
	if (skb_has_frag_list(skb)) {
L
Linus Torvalds 已提交
1587 1588 1589 1590
		skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
		skb_clone_fraglist(n);
	}

1591
	skb_copy_header(n, skb);
L
Linus Torvalds 已提交
1592 1593 1594
out:
	return n;
}
1595
EXPORT_SYMBOL(__pskb_copy_fclone);
L
Linus Torvalds 已提交
1596 1597 1598 1599 1600 1601 1602 1603

/**
 *	pskb_expand_head - reallocate header of &sk_buff
 *	@skb: buffer to reallocate
 *	@nhead: room to add at head
 *	@ntail: room to add at tail
 *	@gfp_mask: allocation priority
 *
1604 1605
 *	Expands (or creates identical copy, if @nhead and @ntail are zero)
 *	header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
L
Linus Torvalds 已提交
1606 1607 1608 1609 1610 1611 1612
 *	reference count of 1. Returns zero in the case of success or error,
 *	if expansion failed. In the last case, &sk_buff is not changed.
 *
 *	All the pointers pointing into skb header may change and must be
 *	reloaded after call to this function.
 */

V
Victor Fusco 已提交
1613
int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1614
		     gfp_t gfp_mask)
L
Linus Torvalds 已提交
1615
{
1616 1617
	int i, osize = skb_end_offset(skb);
	int size = osize + nhead + ntail;
L
Linus Torvalds 已提交
1618
	long off;
1619
	u8 *data;
L
Linus Torvalds 已提交
1620

1621 1622
	BUG_ON(nhead < 0);

1623
	BUG_ON(skb_shared(skb));
L
Linus Torvalds 已提交
1624 1625 1626

	size = SKB_DATA_ALIGN(size);

1627 1628 1629 1630
	if (skb_pfmemalloc(skb))
		gfp_mask |= __GFP_MEMALLOC;
	data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
			       gfp_mask, NUMA_NO_NODE, NULL);
L
Linus Torvalds 已提交
1631 1632
	if (!data)
		goto nodata;
1633
	size = SKB_WITH_OVERHEAD(ksize(data));
L
Linus Torvalds 已提交
1634 1635

	/* Copy only real data... and, alas, header. This should be
E
Eric Dumazet 已提交
1636 1637 1638 1639 1640 1641
	 * optimized for the cases when header is void.
	 */
	memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);

	memcpy((struct skb_shared_info *)(data + size),
	       skb_shinfo(skb),
1642
	       offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
L
Linus Torvalds 已提交
1643

1644 1645 1646 1647 1648 1649
	/*
	 * if shinfo is shared we must drop the old head gracefully, but if it
	 * is not we can just drop the old head and let the existing refcount
	 * be since all we did is relocate the values
	 */
	if (skb_cloned(skb)) {
1650 1651
		if (skb_orphan_frags(skb, gfp_mask))
			goto nofrags;
1652
		if (skb_zcopy(skb))
1653
			refcount_inc(&skb_uarg(skb)->refcnt);
1654
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1655
			skb_frag_ref(skb, i);
L
Linus Torvalds 已提交
1656

1657 1658
		if (skb_has_frag_list(skb))
			skb_clone_fraglist(skb);
L
Linus Torvalds 已提交
1659

1660
		skb_release_data(skb);
1661 1662
	} else {
		skb_free_head(skb);
1663
	}
L
Linus Torvalds 已提交
1664 1665 1666
	off = (data + nhead) - skb->head;

	skb->head     = data;
1667
	skb->head_frag = 0;
L
Linus Torvalds 已提交
1668
	skb->data    += off;
1669 1670
#ifdef NET_SKBUFF_DATA_USES_OFFSET
	skb->end      = size;
1671
	off           = nhead;
1672 1673
#else
	skb->end      = skb->head + size;
1674
#endif
1675
	skb->tail	      += off;
1676
	skb_headers_offset_update(skb, nhead);
L
Linus Torvalds 已提交
1677
	skb->cloned   = 0;
1678
	skb->hdr_len  = 0;
L
Linus Torvalds 已提交
1679 1680
	skb->nohdr    = 0;
	atomic_set(&skb_shinfo(skb)->dataref, 1);
1681

1682 1683
	skb_metadata_clear(skb);

1684 1685 1686 1687 1688 1689 1690
	/* It is not generally safe to change skb->truesize.
	 * For the moment, we really care of rx path, or
	 * when skb is orphaned (not attached to a socket).
	 */
	if (!skb->sk || skb->destructor == sock_edemux)
		skb->truesize += size - osize;

L
Linus Torvalds 已提交
1691 1692
	return 0;

1693 1694
nofrags:
	kfree(data);
L
Linus Torvalds 已提交
1695 1696 1697
nodata:
	return -ENOMEM;
}
1698
EXPORT_SYMBOL(pskb_expand_head);
L
Linus Torvalds 已提交
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718

/* Make private copy of skb with writable head and some headroom */

struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
{
	struct sk_buff *skb2;
	int delta = headroom - skb_headroom(skb);

	if (delta <= 0)
		skb2 = pskb_copy(skb, GFP_ATOMIC);
	else {
		skb2 = skb_clone(skb, GFP_ATOMIC);
		if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
					     GFP_ATOMIC)) {
			kfree_skb(skb2);
			skb2 = NULL;
		}
	}
	return skb2;
}
1719
EXPORT_SYMBOL(skb_realloc_headroom);
L
Linus Torvalds 已提交
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739

/**
 *	skb_copy_expand	-	copy and expand sk_buff
 *	@skb: buffer to copy
 *	@newheadroom: new free bytes at head
 *	@newtailroom: new free bytes at tail
 *	@gfp_mask: allocation priority
 *
 *	Make a copy of both an &sk_buff and its data and while doing so
 *	allocate additional space.
 *
 *	This is used when the caller wishes to modify the data and needs a
 *	private copy of the data to alter as well as more space for new fields.
 *	Returns %NULL on failure or the pointer to the buffer
 *	on success. The returned buffer has a reference count of 1.
 *
 *	You must pass %GFP_ATOMIC as the allocation priority if this function
 *	is called from an interrupt.
 */
struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
V
Victor Fusco 已提交
1740
				int newheadroom, int newtailroom,
1741
				gfp_t gfp_mask)
L
Linus Torvalds 已提交
1742 1743 1744 1745
{
	/*
	 *	Allocate the copy buffer
	 */
1746 1747 1748
	struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
					gfp_mask, skb_alloc_rx_flag(skb),
					NUMA_NO_NODE);
1749
	int oldheadroom = skb_headroom(skb);
L
Linus Torvalds 已提交
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
	int head_copy_len, head_copy_off;

	if (!n)
		return NULL;

	skb_reserve(n, newheadroom);

	/* Set the tail pointer and length */
	skb_put(n, skb->len);

1760
	head_copy_len = oldheadroom;
L
Linus Torvalds 已提交
1761 1762 1763 1764 1765 1766 1767
	head_copy_off = 0;
	if (newheadroom <= head_copy_len)
		head_copy_len = newheadroom;
	else
		head_copy_off = newheadroom - head_copy_len;

	/* Copy the linear header and data. */
1768 1769
	BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
			     skb->len + head_copy_len));
L
Linus Torvalds 已提交
1770

1771
	skb_copy_header(n, skb);
L
Linus Torvalds 已提交
1772

1773
	skb_headers_offset_update(n, newheadroom - oldheadroom);
1774

L
Linus Torvalds 已提交
1775 1776
	return n;
}
1777
EXPORT_SYMBOL(skb_copy_expand);
L
Linus Torvalds 已提交
1778 1779

/**
1780
 *	__skb_pad		-	zero pad the tail of an skb
L
Linus Torvalds 已提交
1781 1782
 *	@skb: buffer to pad
 *	@pad: space to pad
1783
 *	@free_on_error: free buffer on error
L
Linus Torvalds 已提交
1784 1785 1786 1787 1788
 *
 *	Ensure that a buffer is followed by a padding area that is zero
 *	filled. Used by network drivers which may DMA or transfer data
 *	beyond the buffer end onto the wire.
 *
1789 1790
 *	May return error in out of memory cases. The skb is freed on error
 *	if @free_on_error is true.
L
Linus Torvalds 已提交
1791
 */
1792

1793
int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
L
Linus Torvalds 已提交
1794
{
1795 1796
	int err;
	int ntail;
1797

L
Linus Torvalds 已提交
1798
	/* If the skbuff is non linear tailroom is always zero.. */
1799
	if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
L
Linus Torvalds 已提交
1800
		memset(skb->data+skb->len, 0, pad);
1801
		return 0;
L
Linus Torvalds 已提交
1802
	}
1803

1804
	ntail = skb->data_len + pad - (skb->end - skb->tail);
1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821
	if (likely(skb_cloned(skb) || ntail > 0)) {
		err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
		if (unlikely(err))
			goto free_skb;
	}

	/* FIXME: The use of this function with non-linear skb's really needs
	 * to be audited.
	 */
	err = skb_linearize(skb);
	if (unlikely(err))
		goto free_skb;

	memset(skb->data + skb->len, 0, pad);
	return 0;

free_skb:
1822 1823
	if (free_on_error)
		kfree_skb(skb);
1824
	return err;
1825
}
1826
EXPORT_SYMBOL(__skb_pad);
1827

1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840
/**
 *	pskb_put - add data to the tail of a potentially fragmented buffer
 *	@skb: start of the buffer to use
 *	@tail: tail fragment of the buffer to use
 *	@len: amount of data to add
 *
 *	This function extends the used data area of the potentially
 *	fragmented buffer. @tail must be the last fragment of @skb -- or
 *	@skb itself. If this would exceed the total buffer size the kernel
 *	will panic. A pointer to the first byte of the extra data is
 *	returned.
 */

1841
void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1842 1843 1844 1845 1846 1847 1848 1849 1850
{
	if (tail != skb) {
		skb->data_len += len;
		skb->len += len;
	}
	return skb_put(tail, len);
}
EXPORT_SYMBOL_GPL(pskb_put);

1851 1852 1853 1854 1855 1856 1857 1858 1859
/**
 *	skb_put - add data to a buffer
 *	@skb: buffer to use
 *	@len: amount of data to add
 *
 *	This function extends the used data area of the buffer. If this would
 *	exceed the total buffer size the kernel will panic. A pointer to the
 *	first byte of the extra data is returned.
 */
1860
void *skb_put(struct sk_buff *skb, unsigned int len)
1861
{
1862
	void *tmp = skb_tail_pointer(skb);
1863 1864 1865 1866 1867 1868 1869 1870 1871
	SKB_LINEAR_ASSERT(skb);
	skb->tail += len;
	skb->len  += len;
	if (unlikely(skb->tail > skb->end))
		skb_over_panic(skb, len, __builtin_return_address(0));
	return tmp;
}
EXPORT_SYMBOL(skb_put);

1872 1873 1874 1875 1876 1877 1878 1879 1880
/**
 *	skb_push - add data to the start of a buffer
 *	@skb: buffer to use
 *	@len: amount of data to add
 *
 *	This function extends the used data area of the buffer at the buffer
 *	start. If this would exceed the total buffer headroom the kernel will
 *	panic. A pointer to the first byte of the extra data is returned.
 */
1881
void *skb_push(struct sk_buff *skb, unsigned int len)
1882 1883 1884
{
	skb->data -= len;
	skb->len  += len;
1885
	if (unlikely(skb->data < skb->head))
1886 1887 1888 1889 1890
		skb_under_panic(skb, len, __builtin_return_address(0));
	return skb->data;
}
EXPORT_SYMBOL(skb_push);

1891 1892 1893 1894 1895 1896 1897 1898 1899 1900
/**
 *	skb_pull - remove data from the start of a buffer
 *	@skb: buffer to use
 *	@len: amount of data to remove
 *
 *	This function removes data from the start of a buffer, returning
 *	the memory to the headroom. A pointer to the next data in the buffer
 *	is returned. Once the data has been pulled future pushes will overwrite
 *	the old data.
 */
1901
void *skb_pull(struct sk_buff *skb, unsigned int len)
1902
{
1903
	return skb_pull_inline(skb, len);
1904 1905 1906
}
EXPORT_SYMBOL(skb_pull);

1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
/**
 *	skb_trim - remove end from a buffer
 *	@skb: buffer to alter
 *	@len: new length
 *
 *	Cut the length of a buffer down by removing data from the tail. If
 *	the buffer is already under the length specified it is not modified.
 *	The skb must be linear.
 */
void skb_trim(struct sk_buff *skb, unsigned int len)
{
	if (skb->len > len)
		__skb_trim(skb, len);
}
EXPORT_SYMBOL(skb_trim);

1923
/* Trims skb to length len. It can change skb pointers.
L
Linus Torvalds 已提交
1924 1925
 */

1926
int ___pskb_trim(struct sk_buff *skb, unsigned int len)
L
Linus Torvalds 已提交
1927
{
1928 1929
	struct sk_buff **fragp;
	struct sk_buff *frag;
L
Linus Torvalds 已提交
1930 1931 1932
	int offset = skb_headlen(skb);
	int nfrags = skb_shinfo(skb)->nr_frags;
	int i;
1933 1934 1935 1936 1937
	int err;

	if (skb_cloned(skb) &&
	    unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
		return err;
L
Linus Torvalds 已提交
1938

1939 1940 1941 1942 1943
	i = 0;
	if (offset >= len)
		goto drop_pages;

	for (; i < nfrags; i++) {
1944
		int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1945 1946 1947 1948 1949 1950

		if (end < len) {
			offset = end;
			continue;
		}

1951
		skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1952

1953
drop_pages:
1954 1955 1956
		skb_shinfo(skb)->nr_frags = i;

		for (; i < nfrags; i++)
1957
			skb_frag_unref(skb, i);
1958

1959
		if (skb_has_frag_list(skb))
1960
			skb_drop_fraglist(skb);
1961
		goto done;
1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975
	}

	for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
	     fragp = &frag->next) {
		int end = offset + frag->len;

		if (skb_shared(frag)) {
			struct sk_buff *nfrag;

			nfrag = skb_clone(frag, GFP_ATOMIC);
			if (unlikely(!nfrag))
				return -ENOMEM;

			nfrag->next = frag->next;
1976
			consume_skb(frag);
1977 1978
			frag = nfrag;
			*fragp = frag;
L
Linus Torvalds 已提交
1979
		}
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992

		if (end < len) {
			offset = end;
			continue;
		}

		if (end > len &&
		    unlikely((err = pskb_trim(frag, len - offset))))
			return err;

		if (frag->next)
			skb_drop_list(&frag->next);
		break;
L
Linus Torvalds 已提交
1993 1994
	}

1995
done:
1996
	if (len > skb_headlen(skb)) {
L
Linus Torvalds 已提交
1997 1998 1999
		skb->data_len -= skb->len - len;
		skb->len       = len;
	} else {
2000 2001
		skb->len       = len;
		skb->data_len  = 0;
2002
		skb_set_tail_pointer(skb, len);
L
Linus Torvalds 已提交
2003 2004
	}

2005 2006
	if (!skb->sk || skb->destructor == sock_edemux)
		skb_condense(skb);
L
Linus Torvalds 已提交
2007 2008
	return 0;
}
2009
EXPORT_SYMBOL(___pskb_trim);
L
Linus Torvalds 已提交
2010

2011 2012 2013 2014 2015 2016 2017
/* Note : use pskb_trim_rcsum() instead of calling this directly
 */
int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
{
	if (skb->ip_summed == CHECKSUM_COMPLETE) {
		int delta = skb->len - len;

2018 2019 2020
		skb->csum = csum_block_sub(skb->csum,
					   skb_checksum(skb, len, delta, 0),
					   len);
2021 2022 2023 2024 2025 2026
	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
		int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
		int offset = skb_checksum_start_offset(skb) + skb->csum_offset;

		if (offset + sizeof(__sum16) > hdlen)
			return -EINVAL;
2027 2028 2029 2030 2031
	}
	return __pskb_trim(skb, len);
}
EXPORT_SYMBOL(pskb_trim_rcsum_slow);

L
Linus Torvalds 已提交
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056
/**
 *	__pskb_pull_tail - advance tail of skb header
 *	@skb: buffer to reallocate
 *	@delta: number of bytes to advance tail
 *
 *	The function makes a sense only on a fragmented &sk_buff,
 *	it expands header moving its tail forward and copying necessary
 *	data from fragmented part.
 *
 *	&sk_buff MUST have reference count of 1.
 *
 *	Returns %NULL (and &sk_buff does not change) if pull failed
 *	or value of new tail of skb in the case of success.
 *
 *	All the pointers pointing into skb header may change and must be
 *	reloaded after call to this function.
 */

/* Moves tail of skb head forward, copying data from fragmented part,
 * when it is necessary.
 * 1. It may fail due to malloc failure.
 * 2. It may change skb pointers.
 *
 * It is pretty complicated. Luckily, it is called only in exceptional cases.
 */
2057
void *__pskb_pull_tail(struct sk_buff *skb, int delta)
L
Linus Torvalds 已提交
2058 2059 2060 2061 2062
{
	/* If skb has not enough free space at tail, get new one
	 * plus 128 bytes for future expansions. If we have enough
	 * room at tail, reallocate without expansion only if skb is cloned.
	 */
2063
	int i, k, eat = (skb->tail + delta) - skb->end;
L
Linus Torvalds 已提交
2064 2065 2066 2067 2068 2069 2070

	if (eat > 0 || skb_cloned(skb)) {
		if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
				     GFP_ATOMIC))
			return NULL;
	}

2071 2072
	BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
			     skb_tail_pointer(skb), delta));
L
Linus Torvalds 已提交
2073 2074 2075 2076

	/* Optimization: no fragments, no reasons to preestimate
	 * size of pulled pages. Superb.
	 */
2077
	if (!skb_has_frag_list(skb))
L
Linus Torvalds 已提交
2078 2079 2080 2081 2082
		goto pull_pages;

	/* Estimate size of pulled pages. */
	eat = delta;
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2083 2084 2085
		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);

		if (size >= eat)
L
Linus Torvalds 已提交
2086
			goto pull_pages;
2087
		eat -= size;
L
Linus Torvalds 已提交
2088 2089 2090
	}

	/* If we need update frag list, we are in troubles.
W
Wenhua Shi 已提交
2091
	 * Certainly, it is possible to add an offset to skb data,
L
Linus Torvalds 已提交
2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123
	 * but taking into account that pulling is expected to
	 * be very rare operation, it is worth to fight against
	 * further bloating skb head and crucify ourselves here instead.
	 * Pure masohism, indeed. 8)8)
	 */
	if (eat) {
		struct sk_buff *list = skb_shinfo(skb)->frag_list;
		struct sk_buff *clone = NULL;
		struct sk_buff *insp = NULL;

		do {
			if (list->len <= eat) {
				/* Eaten as whole. */
				eat -= list->len;
				list = list->next;
				insp = list;
			} else {
				/* Eaten partially. */

				if (skb_shared(list)) {
					/* Sucks! We need to fork list. :-( */
					clone = skb_clone(list, GFP_ATOMIC);
					if (!clone)
						return NULL;
					insp = list->next;
					list = clone;
				} else {
					/* This may be pulled without
					 * problems. */
					insp = list;
				}
				if (!pskb_pull(list, eat)) {
2124
					kfree_skb(clone);
L
Linus Torvalds 已提交
2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147
					return NULL;
				}
				break;
			}
		} while (eat);

		/* Free pulled out fragments. */
		while ((list = skb_shinfo(skb)->frag_list) != insp) {
			skb_shinfo(skb)->frag_list = list->next;
			kfree_skb(list);
		}
		/* And insert new clone at head. */
		if (clone) {
			clone->next = list;
			skb_shinfo(skb)->frag_list = clone;
		}
	}
	/* Success! Now we may commit changes to skb data. */

pull_pages:
	eat = delta;
	k = 0;
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2148 2149 2150
		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);

		if (size <= eat) {
2151
			skb_frag_unref(skb, i);
2152
			eat -= size;
L
Linus Torvalds 已提交
2153
		} else {
2154 2155 2156
			skb_frag_t *frag = &skb_shinfo(skb)->frags[k];

			*frag = skb_shinfo(skb)->frags[i];
L
Linus Torvalds 已提交
2157
			if (eat) {
2158 2159
				skb_frag_off_add(frag, eat);
				skb_frag_size_sub(frag, eat);
2160 2161
				if (!i)
					goto end;
L
Linus Torvalds 已提交
2162 2163 2164 2165 2166 2167 2168
				eat = 0;
			}
			k++;
		}
	}
	skb_shinfo(skb)->nr_frags = k;

2169
end:
L
Linus Torvalds 已提交
2170 2171 2172
	skb->tail     += delta;
	skb->data_len -= delta;

2173 2174 2175
	if (!skb->data_len)
		skb_zcopy_clear(skb, false);

2176
	return skb_tail_pointer(skb);
L
Linus Torvalds 已提交
2177
}
2178
EXPORT_SYMBOL(__pskb_pull_tail);
L
Linus Torvalds 已提交
2179

2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194
/**
 *	skb_copy_bits - copy bits from skb to kernel buffer
 *	@skb: source skb
 *	@offset: offset in source
 *	@to: destination buffer
 *	@len: number of bytes to copy
 *
 *	Copy the specified number of bytes from the source skb to the
 *	destination buffer.
 *
 *	CAUTION ! :
 *		If its prototype is ever changed,
 *		check arch/{*}/net/{*}.S files,
 *		since it is called from BPF assembly code.
 */
L
Linus Torvalds 已提交
2195 2196
int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
{
2197
	int start = skb_headlen(skb);
2198 2199
	struct sk_buff *frag_iter;
	int i, copy;
L
Linus Torvalds 已提交
2200 2201 2202 2203 2204

	if (offset > (int)skb->len - len)
		goto fault;

	/* Copy header. */
2205
	if ((copy = start - offset) > 0) {
L
Linus Torvalds 已提交
2206 2207
		if (copy > len)
			copy = len;
2208
		skb_copy_from_linear_data_offset(skb, offset, to, copy);
L
Linus Torvalds 已提交
2209 2210 2211 2212 2213 2214 2215
		if ((len -= copy) == 0)
			return 0;
		offset += copy;
		to     += copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2216
		int end;
2217
		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
L
Linus Torvalds 已提交
2218

2219
		WARN_ON(start > offset + len);
2220

2221
		end = start + skb_frag_size(f);
L
Linus Torvalds 已提交
2222
		if ((copy = end - offset) > 0) {
2223 2224
			u32 p_off, p_len, copied;
			struct page *p;
L
Linus Torvalds 已提交
2225 2226 2227 2228 2229
			u8 *vaddr;

			if (copy > len)
				copy = len;

2230
			skb_frag_foreach_page(f,
2231
					      skb_frag_off(f) + offset - start,
2232 2233 2234 2235 2236
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
				memcpy(to + copied, vaddr + p_off, p_len);
				kunmap_atomic(vaddr);
			}
L
Linus Torvalds 已提交
2237 2238 2239 2240 2241 2242

			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			to     += copy;
		}
2243
		start = end;
L
Linus Torvalds 已提交
2244 2245
	}

2246 2247
	skb_walk_frags(skb, frag_iter) {
		int end;
L
Linus Torvalds 已提交
2248

2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			if (copy > len)
				copy = len;
			if (skb_copy_bits(frag_iter, offset - start, to, copy))
				goto fault;
			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			to     += copy;
L
Linus Torvalds 已提交
2261
		}
2262
		start = end;
L
Linus Torvalds 已提交
2263
	}
2264

L
Linus Torvalds 已提交
2265 2266 2267 2268 2269 2270
	if (!len)
		return 0;

fault:
	return -EFAULT;
}
2271
EXPORT_SYMBOL(skb_copy_bits);
L
Linus Torvalds 已提交
2272

2273 2274 2275 2276 2277 2278
/*
 * Callback from splice_to_pipe(), if we need to release some pages
 * at the end of the spd in case we error'ed out in filling the pipe.
 */
static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
2279 2280
	put_page(spd->pages[i]);
}
2281

2282 2283
static struct page *linear_to_page(struct page *page, unsigned int *len,
				   unsigned int *offset,
2284
				   struct sock *sk)
2285
{
2286
	struct page_frag *pfrag = sk_page_frag(sk);
2287

2288 2289
	if (!sk_page_frag_refill(sk, pfrag))
		return NULL;
2290

2291
	*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2292

2293 2294 2295 2296
	memcpy(page_address(pfrag->page) + pfrag->offset,
	       page_address(page) + *offset, *len);
	*offset = pfrag->offset;
	pfrag->offset += *len;
2297

2298
	return pfrag->page;
2299 2300
}

2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
			     struct page *page,
			     unsigned int offset)
{
	return	spd->nr_pages &&
		spd->pages[spd->nr_pages - 1] == page &&
		(spd->partial[spd->nr_pages - 1].offset +
		 spd->partial[spd->nr_pages - 1].len == offset);
}

2311 2312 2313
/*
 * Fill page/offset/length into spd, if it can hold more pages.
 */
2314 2315 2316
static bool spd_fill_page(struct splice_pipe_desc *spd,
			  struct pipe_inode_info *pipe, struct page *page,
			  unsigned int *len, unsigned int offset,
2317
			  bool linear,
2318
			  struct sock *sk)
2319
{
2320
	if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2321
		return true;
2322

2323
	if (linear) {
2324
		page = linear_to_page(page, len, &offset, sk);
2325
		if (!page)
2326
			return true;
2327 2328 2329
	}
	if (spd_can_coalesce(spd, page, offset)) {
		spd->partial[spd->nr_pages - 1].len += *len;
2330
		return false;
2331 2332
	}
	get_page(page);
2333
	spd->pages[spd->nr_pages] = page;
2334
	spd->partial[spd->nr_pages].len = *len;
2335 2336
	spd->partial[spd->nr_pages].offset = offset;
	spd->nr_pages++;
2337

2338
	return false;
2339 2340
}

2341 2342
static bool __splice_segment(struct page *page, unsigned int poff,
			     unsigned int plen, unsigned int *off,
2343
			     unsigned int *len,
2344
			     struct splice_pipe_desc *spd, bool linear,
2345 2346
			     struct sock *sk,
			     struct pipe_inode_info *pipe)
2347
{
2348
	if (!*len)
2349
		return true;
2350 2351 2352 2353

	/* skip this segment if already processed */
	if (*off >= plen) {
		*off -= plen;
2354
		return false;
2355
	}
2356

2357
	/* ignore any bits we already processed */
2358 2359 2360
	poff += *off;
	plen -= *off;
	*off = 0;
2361

2362 2363
	do {
		unsigned int flen = min(*len, plen);
2364

2365 2366 2367 2368 2369 2370 2371
		if (spd_fill_page(spd, pipe, page, &flen, poff,
				  linear, sk))
			return true;
		poff += flen;
		plen -= flen;
		*len -= flen;
	} while (*len && plen);
2372

2373
	return false;
2374 2375 2376
}

/*
2377
 * Map linear and fragment data from the skb to spd. It reports true if the
2378 2379
 * pipe is full or if we already spliced the requested length.
 */
2380 2381 2382
static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
			      unsigned int *offset, unsigned int *len,
			      struct splice_pipe_desc *spd, struct sock *sk)
2383 2384
{
	int seg;
2385
	struct sk_buff *iter;
2386

2387
	/* map the linear part :
2388 2389 2390
	 * If skb->head_frag is set, this 'linear' part is backed by a
	 * fragment, and if the head is not shared with any clones then
	 * we can avoid a copy since we own the head portion of this page.
2391 2392 2393 2394
	 */
	if (__splice_segment(virt_to_page(skb->data),
			     (unsigned long) skb->data & (PAGE_SIZE - 1),
			     skb_headlen(skb),
2395
			     offset, len, spd,
2396
			     skb_head_is_locked(skb),
2397
			     sk, pipe))
2398
		return true;
2399 2400 2401 2402 2403 2404 2405

	/*
	 * then map the fragments
	 */
	for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
		const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];

2406
		if (__splice_segment(skb_frag_page(f),
2407
				     skb_frag_off(f), skb_frag_size(f),
2408
				     offset, len, spd, false, sk, pipe))
2409
			return true;
2410 2411
	}

2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
	skb_walk_frags(skb, iter) {
		if (*offset >= iter->len) {
			*offset -= iter->len;
			continue;
		}
		/* __skb_splice_bits() only fails if the output has no room
		 * left, so no point in going over the frag_list for the error
		 * case.
		 */
		if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
			return true;
	}

2425
	return false;
2426 2427 2428 2429
}

/*
 * Map data from the skb to a pipe. Should handle both the linear part,
2430
 * the fragments, and the frag list.
2431
 */
2432
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2433
		    struct pipe_inode_info *pipe, unsigned int tlen,
2434
		    unsigned int flags)
2435
{
2436 2437
	struct partial_page partial[MAX_SKB_FRAGS];
	struct page *pages[MAX_SKB_FRAGS];
2438 2439 2440
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
2441
		.nr_pages_max = MAX_SKB_FRAGS,
2442
		.ops = &nosteal_pipe_buf_ops,
2443 2444
		.spd_release = sock_spd_release,
	};
2445 2446
	int ret = 0;

2447
	__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
2448

2449
	if (spd.nr_pages)
2450
		ret = splice_to_pipe(pipe, &spd);
2451

2452
	return ret;
2453
}
2454
EXPORT_SYMBOL_GPL(skb_splice_bits);
2455

2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473
/* Send skb data on a socket. Socket must be locked. */
int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
			 int len)
{
	unsigned int orig_len = len;
	struct sk_buff *head = skb;
	unsigned short fragidx;
	int slen, ret;

do_frag_list:

	/* Deal with head data */
	while (offset < skb_headlen(skb) && len) {
		struct kvec kv;
		struct msghdr msg;

		slen = min_t(int, len, skb_headlen(skb) - offset);
		kv.iov_base = skb->data + offset;
2474
		kv.iov_len = slen;
2475
		memset(&msg, 0, sizeof(msg));
2476
		msg.msg_flags = MSG_DONTWAIT;
2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496

		ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
		if (ret <= 0)
			goto error;

		offset += ret;
		len -= ret;
	}

	/* All the data was skb head? */
	if (!len)
		goto out;

	/* Make offset relative to start of frags */
	offset -= skb_headlen(skb);

	/* Find where we are in frag list */
	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];

2497
		if (offset < skb_frag_size(frag))
2498 2499
			break;

2500
		offset -= skb_frag_size(frag);
2501 2502 2503 2504 2505
	}

	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];

2506
		slen = min_t(size_t, len, skb_frag_size(frag) - offset);
2507 2508

		while (slen) {
2509
			ret = kernel_sendpage_locked(sk, skb_frag_page(frag),
2510
						     skb_frag_off(frag) + offset,
2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544
						     slen, MSG_DONTWAIT);
			if (ret <= 0)
				goto error;

			len -= ret;
			offset += ret;
			slen -= ret;
		}

		offset = 0;
	}

	if (len) {
		/* Process any frag lists */

		if (skb == head) {
			if (skb_has_frag_list(skb)) {
				skb = skb_shinfo(skb)->frag_list;
				goto do_frag_list;
			}
		} else if (skb->next) {
			skb = skb->next;
			goto do_frag_list;
		}
	}

out:
	return orig_len - len;

error:
	return orig_len == len ? ret : orig_len - len;
}
EXPORT_SYMBOL_GPL(skb_send_sock_locked);

2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556
/**
 *	skb_store_bits - store bits from kernel buffer to skb
 *	@skb: destination buffer
 *	@offset: offset in destination
 *	@from: source buffer
 *	@len: number of bytes to copy
 *
 *	Copy the specified number of bytes from the source buffer to the
 *	destination skb.  This function handles all the messy bits of
 *	traversing fragment lists and such.
 */

2557
int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2558
{
2559
	int start = skb_headlen(skb);
2560 2561
	struct sk_buff *frag_iter;
	int i, copy;
2562 2563 2564 2565

	if (offset > (int)skb->len - len)
		goto fault;

2566
	if ((copy = start - offset) > 0) {
2567 2568
		if (copy > len)
			copy = len;
2569
		skb_copy_to_linear_data_offset(skb, offset, from, copy);
2570 2571 2572 2573 2574 2575 2576 2577
		if ((len -= copy) == 0)
			return 0;
		offset += copy;
		from += copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2578 2579
		int end;

2580
		WARN_ON(start > offset + len);
2581

2582
		end = start + skb_frag_size(frag);
2583
		if ((copy = end - offset) > 0) {
2584 2585
			u32 p_off, p_len, copied;
			struct page *p;
2586 2587 2588 2589 2590
			u8 *vaddr;

			if (copy > len)
				copy = len;

2591
			skb_frag_foreach_page(frag,
2592
					      skb_frag_off(frag) + offset - start,
2593 2594 2595 2596 2597
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
				memcpy(vaddr + p_off, from + copied, p_len);
				kunmap_atomic(vaddr);
			}
2598 2599 2600 2601 2602 2603

			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			from += copy;
		}
2604
		start = end;
2605 2606
	}

2607 2608
	skb_walk_frags(skb, frag_iter) {
		int end;
2609

2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			if (copy > len)
				copy = len;
			if (skb_store_bits(frag_iter, offset - start,
					   from, copy))
				goto fault;
			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			from += copy;
2623
		}
2624
		start = end;
2625 2626 2627 2628 2629 2630 2631 2632 2633
	}
	if (!len)
		return 0;

fault:
	return -EFAULT;
}
EXPORT_SYMBOL(skb_store_bits);

L
Linus Torvalds 已提交
2634
/* Checksum skb data. */
2635 2636
__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
		      __wsum csum, const struct skb_checksum_ops *ops)
L
Linus Torvalds 已提交
2637
{
2638 2639
	int start = skb_headlen(skb);
	int i, copy = start - offset;
2640
	struct sk_buff *frag_iter;
L
Linus Torvalds 已提交
2641 2642 2643 2644 2645 2646
	int pos = 0;

	/* Checksum header. */
	if (copy > 0) {
		if (copy > len)
			copy = len;
2647 2648
		csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
				       skb->data + offset, copy, csum);
L
Linus Torvalds 已提交
2649 2650 2651 2652 2653 2654 2655
		if ((len -= copy) == 0)
			return csum;
		offset += copy;
		pos	= copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2656
		int end;
2657
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2658

2659
		WARN_ON(start > offset + len);
L
Linus Torvalds 已提交
2660

2661
		end = start + skb_frag_size(frag);
L
Linus Torvalds 已提交
2662
		if ((copy = end - offset) > 0) {
2663 2664
			u32 p_off, p_len, copied;
			struct page *p;
2665
			__wsum csum2;
L
Linus Torvalds 已提交
2666 2667 2668 2669
			u8 *vaddr;

			if (copy > len)
				copy = len;
2670 2671

			skb_frag_foreach_page(frag,
2672
					      skb_frag_off(frag) + offset - start,
2673 2674
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
2675 2676 2677
				csum2 = INDIRECT_CALL_1(ops->update,
							csum_partial_ext,
							vaddr + p_off, p_len, 0);
2678
				kunmap_atomic(vaddr);
2679 2680 2681
				csum = INDIRECT_CALL_1(ops->combine,
						       csum_block_add_ext, csum,
						       csum2, pos, p_len);
2682 2683 2684
				pos += p_len;
			}

L
Linus Torvalds 已提交
2685 2686 2687 2688
			if (!(len -= copy))
				return csum;
			offset += copy;
		}
2689
		start = end;
L
Linus Torvalds 已提交
2690 2691
	}

2692 2693
	skb_walk_frags(skb, frag_iter) {
		int end;
L
Linus Torvalds 已提交
2694

2695 2696 2697 2698 2699 2700 2701
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			__wsum csum2;
			if (copy > len)
				copy = len;
2702 2703
			csum2 = __skb_checksum(frag_iter, offset - start,
					       copy, 0, ops);
2704 2705
			csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
					       csum, csum2, pos, copy);
2706 2707 2708 2709
			if ((len -= copy) == 0)
				return csum;
			offset += copy;
			pos    += copy;
L
Linus Torvalds 已提交
2710
		}
2711
		start = end;
L
Linus Torvalds 已提交
2712
	}
2713
	BUG_ON(len);
L
Linus Torvalds 已提交
2714 2715 2716

	return csum;
}
2717 2718 2719 2720 2721 2722
EXPORT_SYMBOL(__skb_checksum);

__wsum skb_checksum(const struct sk_buff *skb, int offset,
		    int len, __wsum csum)
{
	const struct skb_checksum_ops ops = {
2723
		.update  = csum_partial_ext,
2724 2725 2726 2727 2728
		.combine = csum_block_add_ext,
	};

	return __skb_checksum(skb, offset, len, csum, &ops);
}
2729
EXPORT_SYMBOL(skb_checksum);
L
Linus Torvalds 已提交
2730 2731 2732

/* Both of above in one bottle. */

2733
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2734
				    u8 *to, int len)
L
Linus Torvalds 已提交
2735
{
2736 2737
	int start = skb_headlen(skb);
	int i, copy = start - offset;
2738
	struct sk_buff *frag_iter;
L
Linus Torvalds 已提交
2739
	int pos = 0;
2740
	__wsum csum = 0;
L
Linus Torvalds 已提交
2741 2742 2743 2744 2745 2746

	/* Copy header. */
	if (copy > 0) {
		if (copy > len)
			copy = len;
		csum = csum_partial_copy_nocheck(skb->data + offset, to,
2747
						 copy);
L
Linus Torvalds 已提交
2748 2749 2750 2751 2752 2753 2754 2755
		if ((len -= copy) == 0)
			return csum;
		offset += copy;
		to     += copy;
		pos	= copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2756 2757
		int end;

2758
		WARN_ON(start > offset + len);
L
Linus Torvalds 已提交
2759

2760
		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
L
Linus Torvalds 已提交
2761
		if ((copy = end - offset) > 0) {
2762 2763 2764
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
			u32 p_off, p_len, copied;
			struct page *p;
2765
			__wsum csum2;
L
Linus Torvalds 已提交
2766 2767 2768 2769
			u8 *vaddr;

			if (copy > len)
				copy = len;
2770 2771

			skb_frag_foreach_page(frag,
2772
					      skb_frag_off(frag) + offset - start,
2773 2774 2775 2776
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
				csum2 = csum_partial_copy_nocheck(vaddr + p_off,
								  to + copied,
2777
								  p_len);
2778 2779 2780 2781 2782
				kunmap_atomic(vaddr);
				csum = csum_block_add(csum, csum2, pos);
				pos += p_len;
			}

L
Linus Torvalds 已提交
2783 2784 2785 2786 2787
			if (!(len -= copy))
				return csum;
			offset += copy;
			to     += copy;
		}
2788
		start = end;
L
Linus Torvalds 已提交
2789 2790
	}

2791 2792 2793
	skb_walk_frags(skb, frag_iter) {
		__wsum csum2;
		int end;
L
Linus Torvalds 已提交
2794

2795 2796 2797 2798 2799 2800 2801 2802
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			if (copy > len)
				copy = len;
			csum2 = skb_copy_and_csum_bits(frag_iter,
						       offset - start,
2803
						       to, copy);
2804 2805 2806 2807 2808 2809
			csum = csum_block_add(csum, csum2, pos);
			if ((len -= copy) == 0)
				return csum;
			offset += copy;
			to     += copy;
			pos    += copy;
L
Linus Torvalds 已提交
2810
		}
2811
		start = end;
L
Linus Torvalds 已提交
2812
	}
2813
	BUG_ON(len);
L
Linus Torvalds 已提交
2814 2815
	return csum;
}
2816
EXPORT_SYMBOL(skb_copy_and_csum_bits);
L
Linus Torvalds 已提交
2817

2818 2819 2820 2821 2822
__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
{
	__sum16 sum;

	sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
2823
	/* See comments in __skb_checksum_complete(). */
2824 2825 2826
	if (likely(!sum)) {
		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
		    !skb->csum_complete_sw)
2827
			netdev_rx_csum_fault(skb->dev, skb);
2828 2829 2830 2831 2832 2833 2834
	}
	if (!skb_shared(skb))
		skb->csum_valid = !sum;
	return sum;
}
EXPORT_SYMBOL(__skb_checksum_complete_head);

2835 2836 2837 2838 2839 2840 2841 2842 2843
/* This function assumes skb->csum already holds pseudo header's checksum,
 * which has been changed from the hardware checksum, for example, by
 * __skb_checksum_validate_complete(). And, the original skb->csum must
 * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
 *
 * It returns non-zero if the recomputed checksum is still invalid, otherwise
 * zero. The new checksum is stored back into skb->csum unless the skb is
 * shared.
 */
2844 2845 2846 2847 2848 2849 2850 2851
__sum16 __skb_checksum_complete(struct sk_buff *skb)
{
	__wsum csum;
	__sum16 sum;

	csum = skb_checksum(skb, 0, skb->len, 0);

	sum = csum_fold(csum_add(skb->csum, csum));
2852 2853 2854 2855 2856 2857 2858
	/* This check is inverted, because we already knew the hardware
	 * checksum is invalid before calling this function. So, if the
	 * re-computed checksum is valid instead, then we have a mismatch
	 * between the original skb->csum and skb_checksum(). This means either
	 * the original hardware checksum is incorrect or we screw up skb->csum
	 * when moving skb->data around.
	 */
2859 2860 2861
	if (likely(!sum)) {
		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
		    !skb->csum_complete_sw)
2862
			netdev_rx_csum_fault(skb->dev, skb);
2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
	}

	if (!skb_shared(skb)) {
		/* Save full packet checksum */
		skb->csum = csum;
		skb->ip_summed = CHECKSUM_COMPLETE;
		skb->csum_complete_sw = 1;
		skb->csum_valid = !sum;
	}

	return sum;
}
EXPORT_SYMBOL(__skb_checksum_complete);

2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
{
	net_warn_ratelimited(
		"%s: attempt to compute crc32c without libcrc32c.ko\n",
		__func__);
	return 0;
}

static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
				       int offset, int len)
{
	net_warn_ratelimited(
		"%s: attempt to compute crc32c without libcrc32c.ko\n",
		__func__);
	return 0;
}

static const struct skb_checksum_ops default_crc32c_ops = {
	.update  = warn_crc32c_csum_update,
	.combine = warn_crc32c_csum_combine,
};

const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
	&default_crc32c_ops;
EXPORT_SYMBOL(crc32c_csum_stub);

2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
 /**
 *	skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
 *	@from: source buffer
 *
 *	Calculates the amount of linear headroom needed in the 'to' skb passed
 *	into skb_zerocopy().
 */
unsigned int
skb_zerocopy_headlen(const struct sk_buff *from)
{
	unsigned int hlen = 0;

	if (!from->head_frag ||
	    skb_headlen(from) < L1_CACHE_BYTES ||
	    skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
		hlen = skb_headlen(from);

	if (skb_has_frag_list(from))
		hlen = from->len;

	return hlen;
}
EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);

/**
 *	skb_zerocopy - Zero copy skb to skb
 *	@to: destination buffer
2930
 *	@from: source buffer
2931 2932 2933 2934 2935 2936 2937 2938
 *	@len: number of bytes to copy from source buffer
 *	@hlen: size of linear headroom in destination buffer
 *
 *	Copies up to `len` bytes from `from` to `to` by creating references
 *	to the frags in the source buffer.
 *
 *	The `hlen` as calculated by skb_zerocopy_headlen() specifies the
 *	headroom in the `to` buffer.
2939 2940 2941 2942 2943
 *
 *	Return value:
 *	0: everything is OK
 *	-ENOMEM: couldn't orphan frags of @from due to lack of memory
 *	-EFAULT: skb_copy_bits() found some problem with skb geometry
2944
 */
2945 2946
int
skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2947 2948 2949
{
	int i, j = 0;
	int plen = 0; /* length of skb->head fragment */
2950
	int ret;
2951 2952 2953 2954 2955 2956
	struct page *page;
	unsigned int offset;

	BUG_ON(!from->head_frag && !hlen);

	/* dont bother with small payloads */
2957 2958
	if (len <= skb_tailroom(to))
		return skb_copy_bits(from, 0, skb_put(to, len), len);
2959 2960

	if (hlen) {
2961 2962 2963
		ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
		if (unlikely(ret))
			return ret;
2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980
		len -= hlen;
	} else {
		plen = min_t(int, skb_headlen(from), len);
		if (plen) {
			page = virt_to_head_page(from->head);
			offset = from->data - (unsigned char *)page_address(page);
			__skb_fill_page_desc(to, 0, page, offset, plen);
			get_page(page);
			j = 1;
			len -= plen;
		}
	}

	to->truesize += len + plen;
	to->len += len + plen;
	to->data_len += len + plen;

2981 2982 2983 2984
	if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
		skb_tx_error(from);
		return -ENOMEM;
	}
2985
	skb_zerocopy_clone(to, from, GFP_ATOMIC);
2986

2987
	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2988 2989
		int size;

2990 2991 2992
		if (!len)
			break;
		skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2993 2994 2995 2996
		size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
					len);
		skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
		len -= size;
2997 2998 2999 3000
		skb_frag_ref(to, j);
		j++;
	}
	skb_shinfo(to)->nr_frags = j;
3001 3002

	return 0;
3003 3004 3005
}
EXPORT_SYMBOL_GPL(skb_zerocopy);

L
Linus Torvalds 已提交
3006 3007
void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
{
3008
	__wsum csum;
L
Linus Torvalds 已提交
3009 3010
	long csstart;

3011
	if (skb->ip_summed == CHECKSUM_PARTIAL)
3012
		csstart = skb_checksum_start_offset(skb);
L
Linus Torvalds 已提交
3013 3014 3015
	else
		csstart = skb_headlen(skb);

3016
	BUG_ON(csstart > skb_headlen(skb));
L
Linus Torvalds 已提交
3017

3018
	skb_copy_from_linear_data(skb, to, csstart);
L
Linus Torvalds 已提交
3019 3020 3021 3022

	csum = 0;
	if (csstart != skb->len)
		csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
3023
					      skb->len - csstart);
L
Linus Torvalds 已提交
3024

3025
	if (skb->ip_summed == CHECKSUM_PARTIAL) {
A
Al Viro 已提交
3026
		long csstuff = csstart + skb->csum_offset;
L
Linus Torvalds 已提交
3027

3028
		*((__sum16 *)(to + csstuff)) = csum_fold(csum);
L
Linus Torvalds 已提交
3029 3030
	}
}
3031
EXPORT_SYMBOL(skb_copy_and_csum_dev);
L
Linus Torvalds 已提交
3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051

/**
 *	skb_dequeue - remove from the head of the queue
 *	@list: list to dequeue from
 *
 *	Remove the head of the list. The list lock is taken so the function
 *	may be used safely with other locking list functions. The head item is
 *	returned or %NULL if the list is empty.
 */

struct sk_buff *skb_dequeue(struct sk_buff_head *list)
{
	unsigned long flags;
	struct sk_buff *result;

	spin_lock_irqsave(&list->lock, flags);
	result = __skb_dequeue(list);
	spin_unlock_irqrestore(&list->lock, flags);
	return result;
}
3052
EXPORT_SYMBOL(skb_dequeue);
L
Linus Torvalds 已提交
3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071

/**
 *	skb_dequeue_tail - remove from the tail of the queue
 *	@list: list to dequeue from
 *
 *	Remove the tail of the list. The list lock is taken so the function
 *	may be used safely with other locking list functions. The tail item is
 *	returned or %NULL if the list is empty.
 */
struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
{
	unsigned long flags;
	struct sk_buff *result;

	spin_lock_irqsave(&list->lock, flags);
	result = __skb_dequeue_tail(list);
	spin_unlock_irqrestore(&list->lock, flags);
	return result;
}
3072
EXPORT_SYMBOL(skb_dequeue_tail);
L
Linus Torvalds 已提交
3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087

/**
 *	skb_queue_purge - empty a list
 *	@list: list to empty
 *
 *	Delete all buffers on an &sk_buff list. Each buffer is removed from
 *	the list and one reference dropped. This function takes the list
 *	lock and is atomic with respect to other list locking functions.
 */
void skb_queue_purge(struct sk_buff_head *list)
{
	struct sk_buff *skb;
	while ((skb = skb_dequeue(list)) != NULL)
		kfree_skb(skb);
}
3088
EXPORT_SYMBOL(skb_queue_purge);
L
Linus Torvalds 已提交
3089

3090 3091 3092
/**
 *	skb_rbtree_purge - empty a skb rbtree
 *	@root: root of the rbtree to empty
3093
 *	Return value: the sum of truesizes of all purged skbs.
3094 3095 3096 3097 3098 3099
 *
 *	Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
 *	the list and one reference dropped. This function does not take
 *	any lock. Synchronization should be handled by the caller (e.g., TCP
 *	out-of-order queue is protected by the socket lock).
 */
3100
unsigned int skb_rbtree_purge(struct rb_root *root)
3101
{
3102
	struct rb_node *p = rb_first(root);
3103
	unsigned int sum = 0;
3104

3105 3106
	while (p) {
		struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
3107

3108 3109
		p = rb_next(p);
		rb_erase(&skb->rbnode, root);
3110
		sum += skb->truesize;
3111 3112
		kfree_skb(skb);
	}
3113
	return sum;
3114 3115
}

L
Linus Torvalds 已提交
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134
/**
 *	skb_queue_head - queue a buffer at the list head
 *	@list: list to use
 *	@newsk: buffer to queue
 *
 *	Queue a buffer at the start of the list. This function takes the
 *	list lock and can be used safely with other locking &sk_buff functions
 *	safely.
 *
 *	A buffer cannot be placed on two lists at the same time.
 */
void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
{
	unsigned long flags;

	spin_lock_irqsave(&list->lock, flags);
	__skb_queue_head(list, newsk);
	spin_unlock_irqrestore(&list->lock, flags);
}
3135
EXPORT_SYMBOL(skb_queue_head);
L
Linus Torvalds 已提交
3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155

/**
 *	skb_queue_tail - queue a buffer at the list tail
 *	@list: list to use
 *	@newsk: buffer to queue
 *
 *	Queue a buffer at the tail of the list. This function takes the
 *	list lock and can be used safely with other locking &sk_buff functions
 *	safely.
 *
 *	A buffer cannot be placed on two lists at the same time.
 */
void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
{
	unsigned long flags;

	spin_lock_irqsave(&list->lock, flags);
	__skb_queue_tail(list, newsk);
	spin_unlock_irqrestore(&list->lock, flags);
}
3156
EXPORT_SYMBOL(skb_queue_tail);
D
David S. Miller 已提交
3157

L
Linus Torvalds 已提交
3158 3159 3160
/**
 *	skb_unlink	-	remove a buffer from a list
 *	@skb: buffer to remove
D
David S. Miller 已提交
3161
 *	@list: list to use
L
Linus Torvalds 已提交
3162
 *
D
David S. Miller 已提交
3163 3164
 *	Remove a packet from a list. The list locks are taken and this
 *	function is atomic with respect to other list locked calls
L
Linus Torvalds 已提交
3165
 *
D
David S. Miller 已提交
3166
 *	You must know what list the SKB is on.
L
Linus Torvalds 已提交
3167
 */
D
David S. Miller 已提交
3168
void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
L
Linus Torvalds 已提交
3169
{
D
David S. Miller 已提交
3170
	unsigned long flags;
L
Linus Torvalds 已提交
3171

D
David S. Miller 已提交
3172 3173 3174
	spin_lock_irqsave(&list->lock, flags);
	__skb_unlink(skb, list);
	spin_unlock_irqrestore(&list->lock, flags);
L
Linus Torvalds 已提交
3175
}
3176
EXPORT_SYMBOL(skb_unlink);
L
Linus Torvalds 已提交
3177 3178 3179 3180 3181

/**
 *	skb_append	-	append a buffer
 *	@old: buffer to insert after
 *	@newsk: buffer to insert
D
David S. Miller 已提交
3182
 *	@list: list to use
L
Linus Torvalds 已提交
3183 3184 3185 3186 3187
 *
 *	Place a packet after a given packet in a list. The list locks are taken
 *	and this function is atomic with respect to other list locked calls.
 *	A buffer cannot be placed on two lists at the same time.
 */
D
David S. Miller 已提交
3188
void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
L
Linus Torvalds 已提交
3189 3190 3191
{
	unsigned long flags;

D
David S. Miller 已提交
3192
	spin_lock_irqsave(&list->lock, flags);
3193
	__skb_queue_after(list, old, newsk);
D
David S. Miller 已提交
3194
	spin_unlock_irqrestore(&list->lock, flags);
L
Linus Torvalds 已提交
3195
}
3196
EXPORT_SYMBOL(skb_append);
L
Linus Torvalds 已提交
3197 3198 3199 3200 3201 3202 3203

static inline void skb_split_inside_header(struct sk_buff *skb,
					   struct sk_buff* skb1,
					   const u32 len, const int pos)
{
	int i;

3204 3205
	skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
					 pos - len);
L
Linus Torvalds 已提交
3206 3207 3208 3209 3210 3211 3212 3213 3214 3215
	/* And move data appendix as is. */
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
		skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];

	skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
	skb_shinfo(skb)->nr_frags  = 0;
	skb1->data_len		   = skb->data_len;
	skb1->len		   += skb1->data_len;
	skb->data_len		   = 0;
	skb->len		   = len;
3216
	skb_set_tail_pointer(skb, len);
L
Linus Torvalds 已提交
3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231
}

static inline void skb_split_no_header(struct sk_buff *skb,
				       struct sk_buff* skb1,
				       const u32 len, int pos)
{
	int i, k = 0;
	const int nfrags = skb_shinfo(skb)->nr_frags;

	skb_shinfo(skb)->nr_frags = 0;
	skb1->len		  = skb1->data_len = skb->len - len;
	skb->len		  = len;
	skb->data_len		  = len - pos;

	for (i = 0; i < nfrags; i++) {
3232
		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
L
Linus Torvalds 已提交
3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245

		if (pos + size > len) {
			skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];

			if (pos < len) {
				/* Split frag.
				 * We have two variants in this case:
				 * 1. Move all the frag to the second
				 *    part, if it is possible. F.e.
				 *    this approach is mandatory for TUX,
				 *    where splitting is expensive.
				 * 2. Split is accurately. We make this.
				 */
3246
				skb_frag_ref(skb, i);
3247
				skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
3248 3249
				skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
				skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
L
Linus Torvalds 已提交
3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269
				skb_shinfo(skb)->nr_frags++;
			}
			k++;
		} else
			skb_shinfo(skb)->nr_frags++;
		pos += size;
	}
	skb_shinfo(skb1)->nr_frags = k;
}

/**
 * skb_split - Split fragmented skb to two parts at length len.
 * @skb: the buffer to split
 * @skb1: the buffer to receive the second part
 * @len: new length for skb
 */
void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
{
	int pos = skb_headlen(skb);

3270 3271
	skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
				      SKBTX_SHARED_FRAG;
3272
	skb_zerocopy_clone(skb1, skb, 0);
L
Linus Torvalds 已提交
3273 3274 3275 3276 3277
	if (len < pos)	/* Split line is inside header. */
		skb_split_inside_header(skb, skb1, len, pos);
	else		/* Second chunk has no header, nothing to copy. */
		skb_split_no_header(skb, skb1, len, pos);
}
3278
EXPORT_SYMBOL(skb_split);
L
Linus Torvalds 已提交
3279

3280 3281 3282 3283
/* Shifting from/to a cloned skb is a no-go.
 *
 * Caller cannot keep skb_shinfo related pointers past calling here!
 */
3284 3285
static int skb_prepare_for_shift(struct sk_buff *skb)
{
3286
	return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3287 3288 3289 3290 3291 3292 3293 3294 3295
}

/**
 * skb_shift - Shifts paged data partially from skb to another
 * @tgt: buffer into which tail data gets added
 * @skb: buffer from which the paged data comes from
 * @shiftlen: shift up to this many bytes
 *
 * Attempts to shift up to shiftlen worth of bytes, which may be less than
3296
 * the length of the skb, from skb to tgt. Returns number bytes shifted.
3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
 * It's up to caller to free skb if everything was shifted.
 *
 * If @tgt runs out of frags, the whole operation is aborted.
 *
 * Skb cannot include anything else but paged data while tgt is allowed
 * to have non-paged data as well.
 *
 * TODO: full sized shift could be optimized but that would need
 * specialized skb free'er to handle frags without up-to-date nr_frags.
 */
int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
{
	int from, to, merge, todo;
3310
	skb_frag_t *fragfrom, *fragto;
3311 3312

	BUG_ON(shiftlen > skb->len);
3313 3314 3315

	if (skb_headlen(skb))
		return 0;
3316 3317
	if (skb_zcopy(tgt) || skb_zcopy(skb))
		return 0;
3318 3319 3320 3321 3322 3323 3324 3325 3326 3327

	todo = shiftlen;
	from = 0;
	to = skb_shinfo(tgt)->nr_frags;
	fragfrom = &skb_shinfo(skb)->frags[from];

	/* Actual merge is delayed until the point when we know we can
	 * commit all, so that we don't have to undo partial changes
	 */
	if (!to ||
3328
	    !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3329
			      skb_frag_off(fragfrom))) {
3330 3331 3332 3333
		merge = -1;
	} else {
		merge = to - 1;

3334
		todo -= skb_frag_size(fragfrom);
3335 3336 3337 3338 3339
		if (todo < 0) {
			if (skb_prepare_for_shift(skb) ||
			    skb_prepare_for_shift(tgt))
				return 0;

3340 3341
			/* All previous frag pointers might be stale! */
			fragfrom = &skb_shinfo(skb)->frags[from];
3342 3343
			fragto = &skb_shinfo(tgt)->frags[merge];

3344 3345
			skb_frag_size_add(fragto, shiftlen);
			skb_frag_size_sub(fragfrom, shiftlen);
3346
			skb_frag_off_add(fragfrom, shiftlen);
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368

			goto onlymerged;
		}

		from++;
	}

	/* Skip full, not-fitting skb to avoid expensive operations */
	if ((shiftlen == skb->len) &&
	    (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
		return 0;

	if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
		return 0;

	while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
		if (to == MAX_SKB_FRAGS)
			return 0;

		fragfrom = &skb_shinfo(skb)->frags[from];
		fragto = &skb_shinfo(tgt)->frags[to];

3369
		if (todo >= skb_frag_size(fragfrom)) {
3370
			*fragto = *fragfrom;
3371
			todo -= skb_frag_size(fragfrom);
3372 3373 3374 3375
			from++;
			to++;

		} else {
3376
			__skb_frag_ref(fragfrom);
3377 3378
			skb_frag_page_copy(fragto, fragfrom);
			skb_frag_off_copy(fragto, fragfrom);
3379
			skb_frag_size_set(fragto, todo);
3380

3381
			skb_frag_off_add(fragfrom, todo);
3382
			skb_frag_size_sub(fragfrom, todo);
3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396
			todo = 0;

			to++;
			break;
		}
	}

	/* Ready to "commit" this state change to tgt */
	skb_shinfo(tgt)->nr_frags = to;

	if (merge >= 0) {
		fragfrom = &skb_shinfo(skb)->frags[0];
		fragto = &skb_shinfo(tgt)->frags[merge];

3397
		skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3398
		__skb_frag_unref(fragfrom);
3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426
	}

	/* Reposition in the original skb */
	to = 0;
	while (from < skb_shinfo(skb)->nr_frags)
		skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
	skb_shinfo(skb)->nr_frags = to;

	BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);

onlymerged:
	/* Most likely the tgt won't ever need its checksum anymore, skb on
	 * the other hand might need it if it needs to be resent
	 */
	tgt->ip_summed = CHECKSUM_PARTIAL;
	skb->ip_summed = CHECKSUM_PARTIAL;

	/* Yak, is it really working this way? Some helper please? */
	skb->len -= shiftlen;
	skb->data_len -= shiftlen;
	skb->truesize -= shiftlen;
	tgt->len += shiftlen;
	tgt->data_len += shiftlen;
	tgt->truesize += shiftlen;

	return shiftlen;
}

3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444
/**
 * skb_prepare_seq_read - Prepare a sequential read of skb data
 * @skb: the buffer to read
 * @from: lower offset of data to be read
 * @to: upper offset of data to be read
 * @st: state variable
 *
 * Initializes the specified state variable. Must be called before
 * invoking skb_seq_read() for the first time.
 */
void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
			  unsigned int to, struct skb_seq_state *st)
{
	st->lower_offset = from;
	st->upper_offset = to;
	st->root_skb = st->cur_skb = skb;
	st->frag_idx = st->stepped_offset = 0;
	st->frag_data = NULL;
3445
	st->frag_off = 0;
3446
}
3447
EXPORT_SYMBOL(skb_prepare_seq_read);
3448 3449 3450 3451 3452 3453 3454

/**
 * skb_seq_read - Sequentially read skb data
 * @consumed: number of bytes consumed by the caller so far
 * @data: destination pointer for data to be returned
 * @st: state variable
 *
3455
 * Reads a block of skb data at @consumed relative to the
3456
 * lower offset specified to skb_prepare_seq_read(). Assigns
3457
 * the head of the data block to @data and returns the length
3458 3459 3460 3461
 * of the block or 0 if the end of the skb data or the upper
 * offset has been reached.
 *
 * The caller is not required to consume all of the data
3462
 * returned, i.e. @consumed is typically set to the number
3463 3464 3465
 * of bytes already consumed and the next call to
 * skb_seq_read() will return the remaining part of the block.
 *
L
Lucas De Marchi 已提交
3466
 * Note 1: The size of each block of data returned can be arbitrary,
3467
 *       this limitation is the cost for zerocopy sequential
3468 3469
 *       reads of potentially non linear data.
 *
3470
 * Note 2: Fragment lists within fragments are not implemented
3471 3472 3473 3474 3475 3476 3477 3478 3479
 *       at the moment, state->root_skb could be replaced with
 *       a stack for this purpose.
 */
unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
			  struct skb_seq_state *st)
{
	unsigned int block_limit, abs_offset = consumed + st->lower_offset;
	skb_frag_t *frag;

3480 3481 3482 3483 3484
	if (unlikely(abs_offset >= st->upper_offset)) {
		if (st->frag_data) {
			kunmap_atomic(st->frag_data);
			st->frag_data = NULL;
		}
3485
		return 0;
3486
	}
3487 3488

next_skb:
3489
	block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
3490

3491
	if (abs_offset < block_limit && !st->frag_data) {
3492
		*data = st->cur_skb->data + (abs_offset - st->stepped_offset);
3493 3494 3495 3496 3497 3498 3499
		return block_limit - abs_offset;
	}

	if (st->frag_idx == 0 && !st->frag_data)
		st->stepped_offset += skb_headlen(st->cur_skb);

	while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
3500 3501
		unsigned int pg_idx, pg_off, pg_sz;

3502 3503
		frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];

3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515
		pg_idx = 0;
		pg_off = skb_frag_off(frag);
		pg_sz = skb_frag_size(frag);

		if (skb_frag_must_loop(skb_frag_page(frag))) {
			pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT;
			pg_off = offset_in_page(pg_off + st->frag_off);
			pg_sz = min_t(unsigned int, pg_sz - st->frag_off,
						    PAGE_SIZE - pg_off);
		}

		block_limit = pg_sz + st->stepped_offset;
3516 3517
		if (abs_offset < block_limit) {
			if (!st->frag_data)
3518
				st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx);
3519

3520
			*data = (u8 *)st->frag_data + pg_off +
3521 3522 3523 3524 3525 3526
				(abs_offset - st->stepped_offset);

			return block_limit - abs_offset;
		}

		if (st->frag_data) {
3527
			kunmap_atomic(st->frag_data);
3528 3529 3530
			st->frag_data = NULL;
		}

3531 3532 3533 3534 3535 3536
		st->stepped_offset += pg_sz;
		st->frag_off += pg_sz;
		if (st->frag_off == skb_frag_size(frag)) {
			st->frag_off = 0;
			st->frag_idx++;
		}
3537 3538
	}

3539
	if (st->frag_data) {
3540
		kunmap_atomic(st->frag_data);
3541 3542 3543
		st->frag_data = NULL;
	}

3544
	if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
3545
		st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
3546 3547
		st->frag_idx = 0;
		goto next_skb;
3548 3549
	} else if (st->cur_skb->next) {
		st->cur_skb = st->cur_skb->next;
3550
		st->frag_idx = 0;
3551 3552 3553 3554 3555
		goto next_skb;
	}

	return 0;
}
3556
EXPORT_SYMBOL(skb_seq_read);
3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567

/**
 * skb_abort_seq_read - Abort a sequential read of skb data
 * @st: state variable
 *
 * Must be called if skb_seq_read() was not called until it
 * returned 0.
 */
void skb_abort_seq_read(struct skb_seq_state *st)
{
	if (st->frag_data)
3568
		kunmap_atomic(st->frag_data);
3569
}
3570
EXPORT_SYMBOL(skb_abort_seq_read);
3571

3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598
#define TS_SKB_CB(state)	((struct skb_seq_state *) &((state)->cb))

static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
					  struct ts_config *conf,
					  struct ts_state *state)
{
	return skb_seq_read(offset, text, TS_SKB_CB(state));
}

static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
{
	skb_abort_seq_read(TS_SKB_CB(state));
}

/**
 * skb_find_text - Find a text pattern in skb data
 * @skb: the buffer to look in
 * @from: search offset
 * @to: search limit
 * @config: textsearch configuration
 *
 * Finds a pattern in the skb data according to the specified
 * textsearch configuration. Use textsearch_next() to retrieve
 * subsequent occurrences of the pattern. Returns the offset
 * to the first occurrence or UINT_MAX if no match was found.
 */
unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
3599
			   unsigned int to, struct ts_config *config)
3600
{
3601
	struct ts_state state;
3602 3603
	unsigned int ret;

3604 3605 3606
	config->get_next_block = skb_ts_get_next_block;
	config->finish = skb_ts_finish;

3607
	skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
3608

3609
	ret = textsearch_find(config, &state);
3610
	return (ret <= to - from ? ret : UINT_MAX);
3611
}
3612
EXPORT_SYMBOL(skb_find_text);
3613

3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631
int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
			 int offset, size_t size)
{
	int i = skb_shinfo(skb)->nr_frags;

	if (skb_can_coalesce(skb, i, page, offset)) {
		skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
	} else if (i < MAX_SKB_FRAGS) {
		get_page(page);
		skb_fill_page_desc(skb, i, page, offset, size);
	} else {
		return -EMSGSIZE;
	}

	return 0;
}
EXPORT_SYMBOL_GPL(skb_append_pagefrags);

3632 3633 3634 3635 3636 3637
/**
 *	skb_pull_rcsum - pull skb and update receive checksum
 *	@skb: buffer to update
 *	@len: length of data pulled
 *
 *	This function performs an skb_pull on the packet and updates
3638
 *	the CHECKSUM_COMPLETE checksum.  It should be used on
3639 3640 3641
 *	receive path processing instead of skb_pull unless you know
 *	that the checksum difference is zero (e.g., a valid IP header)
 *	or you are setting ip_summed to CHECKSUM_NONE.
3642
 */
3643
void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
3644
{
3645 3646
	unsigned char *data = skb->data;

3647
	BUG_ON(len > skb->len);
3648 3649 3650
	__skb_pull(skb, len);
	skb_postpull_rcsum(skb, data, len);
	return skb->data;
3651
}
3652 3653
EXPORT_SYMBOL_GPL(skb_pull_rcsum);

3654 3655 3656 3657 3658 3659
static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
{
	skb_frag_t head_frag;
	struct page *page;

	page = virt_to_head_page(frag_skb->head);
3660
	__skb_frag_set_page(&head_frag, page);
3661 3662
	skb_frag_off_set(&head_frag, frag_skb->data -
			 (unsigned char *)page_address(page));
3663
	skb_frag_size_set(&head_frag, skb_headlen(frag_skb));
3664 3665 3666
	return head_frag;
}

3667 3668 3669 3670 3671 3672 3673 3674 3675
struct sk_buff *skb_segment_list(struct sk_buff *skb,
				 netdev_features_t features,
				 unsigned int offset)
{
	struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
	unsigned int tnl_hlen = skb_tnl_header_len(skb);
	unsigned int delta_truesize = 0;
	unsigned int delta_len = 0;
	struct sk_buff *tail = NULL;
3676 3677
	struct sk_buff *nskb, *tmp;
	int err;
3678 3679 3680 3681 3682 3683 3684 3685 3686

	skb_push(skb, -skb_network_offset(skb) + offset);

	skb_shinfo(skb)->frag_list = NULL;

	do {
		nskb = list_skb;
		list_skb = list_skb->next;

3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698
		err = 0;
		if (skb_shared(nskb)) {
			tmp = skb_clone(nskb, GFP_ATOMIC);
			if (tmp) {
				consume_skb(nskb);
				nskb = tmp;
				err = skb_unclone(nskb, GFP_ATOMIC);
			} else {
				err = -ENOMEM;
			}
		}

3699 3700 3701 3702 3703
		if (!tail)
			skb->next = nskb;
		else
			tail->next = nskb;

3704 3705 3706 3707 3708
		if (unlikely(err)) {
			nskb->next = list_skb;
			goto err_linearize;
		}

3709 3710 3711 3712 3713 3714 3715
		tail = nskb;

		delta_len += nskb->len;
		delta_truesize += nskb->truesize;

		skb_push(nskb, -skb_network_offset(nskb) + offset);

3716
		skb_release_head_state(nskb);
3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775
		 __copy_skb_header(nskb, skb);

		skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
		skb_copy_from_linear_data_offset(skb, -tnl_hlen,
						 nskb->data - tnl_hlen,
						 offset + tnl_hlen);

		if (skb_needs_linearize(nskb, features) &&
		    __skb_linearize(nskb))
			goto err_linearize;

	} while (list_skb);

	skb->truesize = skb->truesize - delta_truesize;
	skb->data_len = skb->data_len - delta_len;
	skb->len = skb->len - delta_len;

	skb_gso_reset(skb);

	skb->prev = tail;

	if (skb_needs_linearize(skb, features) &&
	    __skb_linearize(skb))
		goto err_linearize;

	skb_get(skb);

	return skb;

err_linearize:
	kfree_skb_list(skb->next);
	skb->next = NULL;
	return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL_GPL(skb_segment_list);

int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
{
	if (unlikely(p->len + skb->len >= 65536))
		return -E2BIG;

	if (NAPI_GRO_CB(p)->last == p)
		skb_shinfo(p)->frag_list = skb;
	else
		NAPI_GRO_CB(p)->last->next = skb;

	skb_pull(skb, skb_gro_offset(skb));

	NAPI_GRO_CB(p)->last = skb;
	NAPI_GRO_CB(p)->count++;
	p->data_len += skb->len;
	p->truesize += skb->truesize;
	p->len += skb->len;

	NAPI_GRO_CB(skb)->same_flow = 1;

	return 0;
}

H
Herbert Xu 已提交
3776 3777
/**
 *	skb_segment - Perform protocol segmentation on skb.
3778
 *	@head_skb: buffer to segment
3779
 *	@features: features for the output path (see dev->features)
H
Herbert Xu 已提交
3780 3781
 *
 *	This function performs segmentation on the given skb.  It returns
3782 3783
 *	a pointer to the first in a list of new skbs for the segments.
 *	In case of error it returns ERR_PTR(err).
H
Herbert Xu 已提交
3784
 */
3785 3786
struct sk_buff *skb_segment(struct sk_buff *head_skb,
			    netdev_features_t features)
H
Herbert Xu 已提交
3787 3788 3789
{
	struct sk_buff *segs = NULL;
	struct sk_buff *tail = NULL;
3790
	struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
3791 3792 3793
	skb_frag_t *frag = skb_shinfo(head_skb)->frags;
	unsigned int mss = skb_shinfo(head_skb)->gso_size;
	unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
3794
	struct sk_buff *frag_skb = head_skb;
H
Herbert Xu 已提交
3795
	unsigned int offset = doffset;
3796
	unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
3797
	unsigned int partial_segs = 0;
H
Herbert Xu 已提交
3798
	unsigned int headroom;
3799
	unsigned int len = head_skb->len;
3800
	__be16 proto;
3801
	bool csum, sg;
3802
	int nfrags = skb_shinfo(head_skb)->nr_frags;
H
Herbert Xu 已提交
3803 3804 3805 3806
	int err = -ENOMEM;
	int i = 0;
	int pos;

3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825
	if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) &&
	    (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) {
		/* gso_size is untrusted, and we have a frag_list with a linear
		 * non head_frag head.
		 *
		 * (we assume checking the first list_skb member suffices;
		 * i.e if either of the list_skb members have non head_frag
		 * head, then the first one has too).
		 *
		 * If head_skb's headlen does not fit requested gso_size, it
		 * means that the frag_list members do NOT terminate on exact
		 * gso_size boundaries. Hence we cannot perform skb_frag_t page
		 * sharing. Therefore we must fallback to copying the frag_list
		 * skbs; we do so by disabling SG.
		 */
		if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb))
			features &= ~NETIF_F_SG;
	}

3826
	__skb_push(head_skb, doffset);
3827
	proto = skb_network_protocol(head_skb, NULL);
3828 3829 3830
	if (unlikely(!proto))
		return ERR_PTR(-EINVAL);

3831
	sg = !!(features & NETIF_F_SG);
3832
	csum = !!can_checksum_protocol(features, proto);
3833

3834 3835 3836
	if (sg && csum && (mss != GSO_BY_FRAGS))  {
		if (!(features & NETIF_F_GSO_PARTIAL)) {
			struct sk_buff *iter;
3837
			unsigned int frag_len;
3838 3839 3840 3841 3842

			if (!list_skb ||
			    !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
				goto normal;

3843 3844 3845 3846 3847 3848 3849
			/* If we get here then all the required
			 * GSO features except frag_list are supported.
			 * Try to split the SKB to multiple GSO SKBs
			 * with no frag_list.
			 * Currently we can do that only when the buffers don't
			 * have a linear part and all the buffers except
			 * the last are of the same length.
3850
			 */
3851
			frag_len = list_skb->len;
3852
			skb_walk_frags(head_skb, iter) {
3853 3854
				if (frag_len != iter->len && iter->next)
					goto normal;
3855
				if (skb_headlen(iter) && !iter->head_frag)
3856 3857 3858 3859
					goto normal;

				len -= iter->len;
			}
3860 3861 3862

			if (len != frag_len)
				goto normal;
3863 3864 3865 3866 3867 3868
		}

		/* GSO partial only requires that we trim off any excess that
		 * doesn't fit into an MSS sized block, so take care of that
		 * now.
		 */
3869
		partial_segs = len / mss;
3870 3871 3872 3873
		if (partial_segs > 1)
			mss *= partial_segs;
		else
			partial_segs = 0;
3874 3875
	}

3876
normal:
3877 3878
	headroom = skb_headroom(head_skb);
	pos = skb_headlen(head_skb);
H
Herbert Xu 已提交
3879 3880 3881

	do {
		struct sk_buff *nskb;
3882
		skb_frag_t *nskb_frag;
3883
		int hsize;
H
Herbert Xu 已提交
3884 3885
		int size;

3886 3887 3888 3889 3890 3891 3892
		if (unlikely(mss == GSO_BY_FRAGS)) {
			len = list_skb->len;
		} else {
			len = head_skb->len - offset;
			if (len > mss)
				len = mss;
		}
H
Herbert Xu 已提交
3893

3894
		hsize = skb_headlen(head_skb) - offset;
H
Herbert Xu 已提交
3895 3896
		if (hsize < 0)
			hsize = 0;
3897 3898
		if (hsize > len || !sg)
			hsize = len;
H
Herbert Xu 已提交
3899

3900 3901 3902
		if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
		    (skb_headlen(list_skb) == len || sg)) {
			BUG_ON(skb_headlen(list_skb) > len);
3903 3904

			i = 0;
3905 3906
			nfrags = skb_shinfo(list_skb)->nr_frags;
			frag = skb_shinfo(list_skb)->frags;
3907
			frag_skb = list_skb;
3908
			pos += skb_headlen(list_skb);
3909 3910 3911 3912

			while (pos < offset + len) {
				BUG_ON(i >= nfrags);

3913
				size = skb_frag_size(frag);
3914 3915 3916 3917 3918
				if (pos + size > offset + len)
					break;

				i++;
				pos += size;
3919
				frag++;
3920
			}
3921

3922 3923
			nskb = skb_clone(list_skb, GFP_ATOMIC);
			list_skb = list_skb->next;
3924 3925 3926 3927

			if (unlikely(!nskb))
				goto err;

3928 3929 3930 3931 3932
			if (unlikely(pskb_trim(nskb, len))) {
				kfree_skb(nskb);
				goto err;
			}

3933
			hsize = skb_end_offset(nskb);
3934 3935 3936 3937 3938
			if (skb_cow_head(nskb, doffset + headroom)) {
				kfree_skb(nskb);
				goto err;
			}

3939
			nskb->truesize += skb_end_offset(nskb) - hsize;
3940 3941 3942
			skb_release_head_state(nskb);
			__skb_push(nskb, doffset);
		} else {
3943
			nskb = __alloc_skb(hsize + doffset + headroom,
3944
					   GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
3945
					   NUMA_NO_NODE);
3946 3947 3948 3949 3950 3951 3952

			if (unlikely(!nskb))
				goto err;

			skb_reserve(nskb, headroom);
			__skb_put(nskb, doffset);
		}
H
Herbert Xu 已提交
3953 3954 3955 3956 3957 3958 3959

		if (segs)
			tail->next = nskb;
		else
			segs = nskb;
		tail = nskb;

3960
		__copy_skb_header(nskb, head_skb);
H
Herbert Xu 已提交
3961

3962
		skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
3963
		skb_reset_mac_len(nskb);
3964

3965
		skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
3966 3967
						 nskb->data - tnl_hlen,
						 doffset + tnl_hlen);
3968

3969
		if (nskb->len == len + doffset)
3970
			goto perform_csum_check;
3971

3972
		if (!sg) {
3973 3974 3975 3976 3977 3978 3979
			if (!csum) {
				if (!nskb->remcsum_offload)
					nskb->ip_summed = CHECKSUM_NONE;
				SKB_GSO_CB(nskb)->csum =
					skb_copy_and_csum_bits(head_skb, offset,
							       skb_put(nskb,
								       len),
3980
							       len);
3981 3982 3983 3984 3985 3986 3987
				SKB_GSO_CB(nskb)->csum_start =
					skb_headroom(nskb) + doffset;
			} else {
				skb_copy_bits(head_skb, offset,
					      skb_put(nskb, len),
					      len);
			}
H
Herbert Xu 已提交
3988 3989 3990
			continue;
		}

3991
		nskb_frag = skb_shinfo(nskb)->frags;
H
Herbert Xu 已提交
3992

3993
		skb_copy_from_linear_data_offset(head_skb, offset,
3994
						 skb_put(nskb, hsize), hsize);
H
Herbert Xu 已提交
3995

3996 3997
		skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
					      SKBTX_SHARED_FRAG;
3998

3999 4000 4001 4002
		if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
		    skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
			goto err;

4003 4004 4005
		while (pos < offset + len) {
			if (i >= nfrags) {
				i = 0;
4006 4007
				nfrags = skb_shinfo(list_skb)->nr_frags;
				frag = skb_shinfo(list_skb)->frags;
4008
				frag_skb = list_skb;
4009 4010 4011 4012
				if (!skb_headlen(list_skb)) {
					BUG_ON(!nfrags);
				} else {
					BUG_ON(!list_skb->head_frag);
4013

4014 4015 4016 4017
					/* to make room for head_frag. */
					i--;
					frag--;
				}
4018 4019 4020 4021 4022
				if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
				    skb_zerocopy_clone(nskb, frag_skb,
						       GFP_ATOMIC))
					goto err;

4023
				list_skb = list_skb->next;
4024 4025 4026 4027 4028 4029 4030
			}

			if (unlikely(skb_shinfo(nskb)->nr_frags >=
				     MAX_SKB_FRAGS)) {
				net_warn_ratelimited(
					"skb_segment: too many frags: %u %u\n",
					pos, mss);
4031
				err = -EINVAL;
4032 4033 4034
				goto err;
			}

4035
			*nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
4036 4037
			__skb_frag_ref(nskb_frag);
			size = skb_frag_size(nskb_frag);
H
Herbert Xu 已提交
4038 4039

			if (pos < offset) {
4040
				skb_frag_off_add(nskb_frag, offset - pos);
4041
				skb_frag_size_sub(nskb_frag, offset - pos);
H
Herbert Xu 已提交
4042 4043
			}

4044
			skb_shinfo(nskb)->nr_frags++;
H
Herbert Xu 已提交
4045 4046 4047

			if (pos + size <= offset + len) {
				i++;
4048
				frag++;
H
Herbert Xu 已提交
4049 4050
				pos += size;
			} else {
4051
				skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
4052
				goto skip_fraglist;
H
Herbert Xu 已提交
4053 4054
			}

4055
			nskb_frag++;
H
Herbert Xu 已提交
4056 4057
		}

4058
skip_fraglist:
H
Herbert Xu 已提交
4059 4060 4061
		nskb->data_len = len - hsize;
		nskb->len += nskb->data_len;
		nskb->truesize += nskb->data_len;
4062

4063
perform_csum_check:
4064
		if (!csum) {
4065 4066 4067 4068
			if (skb_has_shared_frag(nskb) &&
			    __skb_linearize(nskb))
				goto err;

4069 4070
			if (!nskb->remcsum_offload)
				nskb->ip_summed = CHECKSUM_NONE;
4071 4072 4073
			SKB_GSO_CB(nskb)->csum =
				skb_checksum(nskb, doffset,
					     nskb->len - doffset, 0);
4074
			SKB_GSO_CB(nskb)->csum_start =
4075
				skb_headroom(nskb) + doffset;
4076
		}
4077
	} while ((offset += len) < head_skb->len);
H
Herbert Xu 已提交
4078

4079 4080 4081 4082 4083
	/* Some callers want to get the end of the list.
	 * Put it in segs->prev to avoid walking the list.
	 * (see validate_xmit_skb_list() for example)
	 */
	segs->prev = tail;
4084

4085
	if (partial_segs) {
4086
		struct sk_buff *iter;
4087
		int type = skb_shinfo(head_skb)->gso_type;
4088
		unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
4089 4090

		/* Update type to add partial and then remove dodgy if set */
4091
		type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
4092 4093 4094 4095 4096
		type &= ~SKB_GSO_DODGY;

		/* Update GSO info and prepare to start updating headers on
		 * our way back down the stack of protocols.
		 */
4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107
		for (iter = segs; iter; iter = iter->next) {
			skb_shinfo(iter)->gso_size = gso_size;
			skb_shinfo(iter)->gso_segs = partial_segs;
			skb_shinfo(iter)->gso_type = type;
			SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
		}

		if (tail->len - doffset <= gso_size)
			skb_shinfo(tail)->gso_size = 0;
		else if (tail != segs)
			skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
4108 4109
	}

4110 4111 4112 4113 4114 4115 4116 4117 4118
	/* Following permits correct backpressure, for protocols
	 * using skb_set_owner_w().
	 * Idea is to tranfert ownership from head_skb to last segment.
	 */
	if (head_skb->destructor == sock_wfree) {
		swap(tail->truesize, head_skb->truesize);
		swap(tail->destructor, head_skb->destructor);
		swap(tail->sk, head_skb->sk);
	}
H
Herbert Xu 已提交
4119 4120 4121
	return segs;

err:
4122
	kfree_skb_list(segs);
H
Herbert Xu 已提交
4123 4124 4125 4126
	return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(skb_segment);

4127
int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
H
Herbert Xu 已提交
4128
{
4129
	struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
4130 4131
	unsigned int offset = skb_gro_offset(skb);
	unsigned int headlen = skb_headlen(skb);
4132
	unsigned int len = skb_gro_len(skb);
4133
	unsigned int delta_truesize;
4134
	struct sk_buff *lp;
H
Herbert Xu 已提交
4135

4136
	if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush))
H
Herbert Xu 已提交
4137 4138
		return -E2BIG;

4139
	lp = NAPI_GRO_CB(p)->last;
4140 4141 4142
	pinfo = skb_shinfo(lp);

	if (headlen <= offset) {
4143
		skb_frag_t *frag;
4144
		skb_frag_t *frag2;
4145 4146
		int i = skbinfo->nr_frags;
		int nr_frags = pinfo->nr_frags + i;
4147 4148

		if (nr_frags > MAX_SKB_FRAGS)
4149
			goto merge;
4150

4151
		offset -= headlen;
4152 4153
		pinfo->nr_frags = nr_frags;
		skbinfo->nr_frags = 0;
4154

4155 4156
		frag = pinfo->frags + nr_frags;
		frag2 = skbinfo->frags + i;
4157 4158 4159
		do {
			*--frag = *--frag2;
		} while (--i);
H
Herbert Xu 已提交
4160

4161
		skb_frag_off_add(frag, offset);
4162
		skb_frag_size_sub(frag, offset);
4163

4164
		/* all fragments truesize : remove (head size + sk_buff) */
4165 4166
		delta_truesize = skb->truesize -
				 SKB_TRUESIZE(skb_end_offset(skb));
4167

4168 4169 4170 4171
		skb->truesize -= skb->data_len;
		skb->len -= skb->data_len;
		skb->data_len = 0;

4172
		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
H
Herbert Xu 已提交
4173
		goto done;
4174 4175 4176 4177 4178 4179 4180 4181
	} else if (skb->head_frag) {
		int nr_frags = pinfo->nr_frags;
		skb_frag_t *frag = pinfo->frags + nr_frags;
		struct page *page = virt_to_head_page(skb->head);
		unsigned int first_size = headlen - offset;
		unsigned int first_offset;

		if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
4182
			goto merge;
4183 4184 4185 4186 4187 4188 4189

		first_offset = skb->data -
			       (unsigned char *)page_address(page) +
			       offset;

		pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;

4190
		__skb_frag_set_page(frag, page);
4191
		skb_frag_off_set(frag, first_offset);
4192 4193 4194 4195 4196
		skb_frag_size_set(frag, first_size);

		memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
		/* We dont need to clear skbinfo->nr_frags here */

4197
		delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4198 4199
		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
		goto done;
4200
	}
H
Herbert Xu 已提交
4201 4202

merge:
4203
	delta_truesize = skb->truesize;
4204
	if (offset > headlen) {
4205 4206
		unsigned int eat = offset - headlen;

4207
		skb_frag_off_add(&skbinfo->frags[0], eat);
4208
		skb_frag_size_sub(&skbinfo->frags[0], eat);
4209 4210
		skb->data_len -= eat;
		skb->len -= eat;
4211
		offset = headlen;
4212 4213
	}

4214
	__skb_pull(skb, offset);
4215

4216
	if (NAPI_GRO_CB(p)->last == p)
4217 4218 4219
		skb_shinfo(p)->frag_list = skb;
	else
		NAPI_GRO_CB(p)->last->next = skb;
4220
	NAPI_GRO_CB(p)->last = skb;
4221
	__skb_header_release(skb);
4222
	lp = p;
H
Herbert Xu 已提交
4223

H
Herbert Xu 已提交
4224 4225
done:
	NAPI_GRO_CB(p)->count++;
4226
	p->data_len += len;
4227
	p->truesize += delta_truesize;
4228
	p->len += len;
4229 4230 4231 4232 4233
	if (lp != p) {
		lp->data_len += len;
		lp->truesize += delta_truesize;
		lp->len += len;
	}
H
Herbert Xu 已提交
4234 4235 4236 4237
	NAPI_GRO_CB(skb)->same_flow = 1;
	return 0;
}

4238 4239 4240 4241 4242 4243 4244 4245
#ifdef CONFIG_SKB_EXTENSIONS
#define SKB_EXT_ALIGN_VALUE	8
#define SKB_EXT_CHUNKSIZEOF(x)	(ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)

static const u8 skb_ext_type_len[] = {
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
	[SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
#endif
4246 4247 4248
#ifdef CONFIG_XFRM
	[SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
#endif
4249 4250 4251
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	[TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
#endif
4252 4253 4254
#if IS_ENABLED(CONFIG_MPTCP)
	[SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
#endif
4255 4256 4257 4258 4259 4260 4261
};

static __always_inline unsigned int skb_ext_total_length(void)
{
	return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
		skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
4262 4263 4264
#endif
#ifdef CONFIG_XFRM
		skb_ext_type_len[SKB_EXT_SEC_PATH] +
4265 4266 4267
#endif
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
		skb_ext_type_len[TC_SKB_EXT] +
4268 4269 4270
#endif
#if IS_ENABLED(CONFIG_MPTCP)
		skb_ext_type_len[SKB_EXT_MPTCP] +
4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289
#endif
		0;
}

static void skb_extensions_init(void)
{
	BUILD_BUG_ON(SKB_EXT_NUM >= 8);
	BUILD_BUG_ON(skb_ext_total_length() > 255);

	skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
					     SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
					     0,
					     SLAB_HWCACHE_ALIGN|SLAB_PANIC,
					     NULL);
}
#else
static void skb_extensions_init(void) {}
#endif

L
Linus Torvalds 已提交
4290 4291
void __init skb_init(void)
{
4292
	skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
L
Linus Torvalds 已提交
4293 4294
					      sizeof(struct sk_buff),
					      0,
A
Alexey Dobriyan 已提交
4295
					      SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4296 4297
					      offsetof(struct sk_buff, cb),
					      sizeof_field(struct sk_buff, cb),
4298
					      NULL);
4299
	skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
4300
						sizeof(struct sk_buff_fclones),
4301
						0,
A
Alexey Dobriyan 已提交
4302
						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4303
						NULL);
4304
	skb_extensions_init();
L
Linus Torvalds 已提交
4305 4306
}

4307
static int
4308 4309
__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
	       unsigned int recursion_level)
4310
{
4311 4312
	int start = skb_headlen(skb);
	int i, copy = start - offset;
4313
	struct sk_buff *frag_iter;
4314 4315
	int elt = 0;

4316 4317 4318
	if (unlikely(recursion_level >= 24))
		return -EMSGSIZE;

4319 4320 4321
	if (copy > 0) {
		if (copy > len)
			copy = len;
4322
		sg_set_buf(sg, skb->data + offset, copy);
4323 4324 4325 4326 4327 4328 4329
		elt++;
		if ((len -= copy) == 0)
			return elt;
		offset += copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4330
		int end;
4331

4332
		WARN_ON(start > offset + len);
4333

4334
		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
4335 4336
		if ((copy = end - offset) > 0) {
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4337 4338
			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
				return -EMSGSIZE;
4339 4340 4341

			if (copy > len)
				copy = len;
4342
			sg_set_page(&sg[elt], skb_frag_page(frag), copy,
4343
				    skb_frag_off(frag) + offset - start);
4344 4345 4346 4347 4348
			elt++;
			if (!(len -= copy))
				return elt;
			offset += copy;
		}
4349
		start = end;
4350 4351
	}

4352
	skb_walk_frags(skb, frag_iter) {
4353
		int end, ret;
4354

4355
		WARN_ON(start > offset + len);
4356

4357 4358
		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
4359 4360 4361
			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
				return -EMSGSIZE;

4362 4363
			if (copy > len)
				copy = len;
4364 4365 4366 4367 4368
			ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
					      copy, recursion_level + 1);
			if (unlikely(ret < 0))
				return ret;
			elt += ret;
4369 4370 4371
			if ((len -= copy) == 0)
				return elt;
			offset += copy;
4372
		}
4373
		start = end;
4374 4375 4376 4377 4378
	}
	BUG_ON(len);
	return elt;
}

4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403
/**
 *	skb_to_sgvec - Fill a scatter-gather list from a socket buffer
 *	@skb: Socket buffer containing the buffers to be mapped
 *	@sg: The scatter-gather list to map into
 *	@offset: The offset into the buffer's contents to start mapping
 *	@len: Length of buffer space to be mapped
 *
 *	Fill the specified scatter-gather list with mappings/pointers into a
 *	region of the buffer space attached to a socket buffer. Returns either
 *	the number of scatterlist items used, or -EMSGSIZE if the contents
 *	could not fit.
 */
int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
{
	int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);

	if (nsg <= 0)
		return nsg;

	sg_mark_end(&sg[nsg - 1]);

	return nsg;
}
EXPORT_SYMBOL_GPL(skb_to_sgvec);

4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425
/* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
 * sglist without mark the sg which contain last skb data as the end.
 * So the caller can mannipulate sg list as will when padding new data after
 * the first call without calling sg_unmark_end to expend sg list.
 *
 * Scenario to use skb_to_sgvec_nomark:
 * 1. sg_init_table
 * 2. skb_to_sgvec_nomark(payload1)
 * 3. skb_to_sgvec_nomark(payload2)
 *
 * This is equivalent to:
 * 1. sg_init_table
 * 2. skb_to_sgvec(payload1)
 * 3. sg_unmark_end
 * 4. skb_to_sgvec(payload2)
 *
 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
 * is more preferable.
 */
int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
			int offset, int len)
{
4426
	return __skb_to_sgvec(skb, sg, offset, len, 0);
4427 4428 4429
}
EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);

4430 4431


4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459
/**
 *	skb_cow_data - Check that a socket buffer's data buffers are writable
 *	@skb: The socket buffer to check.
 *	@tailbits: Amount of trailing space to be added
 *	@trailer: Returned pointer to the skb where the @tailbits space begins
 *
 *	Make sure that the data buffers attached to a socket buffer are
 *	writable. If they are not, private copies are made of the data buffers
 *	and the socket buffer is set to use these instead.
 *
 *	If @tailbits is given, make sure that there is space to write @tailbits
 *	bytes of data beyond current end of socket buffer.  @trailer will be
 *	set to point to the skb in which this space begins.
 *
 *	The number of scatterlist elements required to completely map the
 *	COW'd and extended socket buffer will be returned.
 */
int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
{
	int copyflag;
	int elt;
	struct sk_buff *skb1, **skb_p;

	/* If skb is cloned or its head is paged, reallocate
	 * head pulling out all the pages (pages are considered not writable
	 * at the moment even if they are anonymous).
	 */
	if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4460
	    !__pskb_pull_tail(skb, __skb_pagelen(skb)))
4461 4462 4463
		return -ENOMEM;

	/* Easy case. Most of packets will go this way. */
4464
	if (!skb_has_frag_list(skb)) {
4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498
		/* A little of trouble, not enough of space for trailer.
		 * This should not happen, when stack is tuned to generate
		 * good frames. OK, on miss we reallocate and reserve even more
		 * space, 128 bytes is fair. */

		if (skb_tailroom(skb) < tailbits &&
		    pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
			return -ENOMEM;

		/* Voila! */
		*trailer = skb;
		return 1;
	}

	/* Misery. We are in troubles, going to mincer fragments... */

	elt = 1;
	skb_p = &skb_shinfo(skb)->frag_list;
	copyflag = 0;

	while ((skb1 = *skb_p) != NULL) {
		int ntail = 0;

		/* The fragment is partially pulled by someone,
		 * this can happen on input. Copy it and everything
		 * after it. */

		if (skb_shared(skb1))
			copyflag = 1;

		/* If the skb is the last, worry about trailer. */

		if (skb1->next == NULL && tailbits) {
			if (skb_shinfo(skb1)->nr_frags ||
4499
			    skb_has_frag_list(skb1) ||
4500 4501 4502 4503 4504 4505 4506 4507
			    skb_tailroom(skb1) < tailbits)
				ntail = tailbits + 128;
		}

		if (copyflag ||
		    skb_cloned(skb1) ||
		    ntail ||
		    skb_shinfo(skb1)->nr_frags ||
4508
		    skb_has_frag_list(skb1)) {
4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539
			struct sk_buff *skb2;

			/* Fuck, we are miserable poor guys... */
			if (ntail == 0)
				skb2 = skb_copy(skb1, GFP_ATOMIC);
			else
				skb2 = skb_copy_expand(skb1,
						       skb_headroom(skb1),
						       ntail,
						       GFP_ATOMIC);
			if (unlikely(skb2 == NULL))
				return -ENOMEM;

			if (skb1->sk)
				skb_set_owner_w(skb2, skb1->sk);

			/* Looking around. Are we still alive?
			 * OK, link new skb, drop old one */

			skb2->next = skb1->next;
			*skb_p = skb2;
			kfree_skb(skb1);
			skb1 = skb2;
		}
		elt++;
		*trailer = skb1;
		skb_p = &skb1->next;
	}

	return elt;
}
4540
EXPORT_SYMBOL_GPL(skb_cow_data);
4541

4542 4543 4544 4545 4546 4547 4548
static void sock_rmem_free(struct sk_buff *skb)
{
	struct sock *sk = skb->sk;

	atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
}

4549 4550 4551 4552 4553 4554 4555 4556 4557
static void skb_set_err_queue(struct sk_buff *skb)
{
	/* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
	 * So, it is safe to (mis)use it to mark skbs on the error queue.
	 */
	skb->pkt_type = PACKET_OUTGOING;
	BUILD_BUG_ON(PACKET_OUTGOING == 0);
}

4558 4559 4560 4561 4562 4563
/*
 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
 */
int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
{
	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4564
	    (unsigned int)READ_ONCE(sk->sk_rcvbuf))
4565 4566 4567 4568 4569 4570
		return -ENOMEM;

	skb_orphan(skb);
	skb->sk = sk;
	skb->destructor = sock_rmem_free;
	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4571
	skb_set_err_queue(skb);
4572

4573 4574 4575
	/* before exiting rcu section, make sure dst is refcounted */
	skb_dst_force(skb);

4576 4577
	skb_queue_tail(&sk->sk_error_queue, skb);
	if (!sock_flag(sk, SOCK_DEAD))
4578
		sk->sk_error_report(sk);
4579 4580 4581 4582
	return 0;
}
EXPORT_SYMBOL(sock_queue_err_skb);

4583 4584 4585 4586 4587 4588
static bool is_icmp_err_skb(const struct sk_buff *skb)
{
	return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
		       SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
}

4589 4590 4591
struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
{
	struct sk_buff_head *q = &sk->sk_error_queue;
4592 4593
	struct sk_buff *skb, *skb_next = NULL;
	bool icmp_next = false;
4594
	unsigned long flags;
4595

4596
	spin_lock_irqsave(&q->lock, flags);
4597
	skb = __skb_dequeue(q);
4598
	if (skb && (skb_next = skb_peek(q))) {
4599
		icmp_next = is_icmp_err_skb(skb_next);
4600
		if (icmp_next)
4601
			sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
4602
	}
4603
	spin_unlock_irqrestore(&q->lock, flags);
4604

4605 4606 4607 4608
	if (is_icmp_err_skb(skb) && !icmp_next)
		sk->sk_err = 0;

	if (skb_next)
4609 4610 4611 4612 4613 4614
		sk->sk_error_report(sk);

	return skb;
}
EXPORT_SYMBOL(sock_dequeue_err_skb);

4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627
/**
 * skb_clone_sk - create clone of skb, and take reference to socket
 * @skb: the skb to clone
 *
 * This function creates a clone of a buffer that holds a reference on
 * sk_refcnt.  Buffers created via this function are meant to be
 * returned using sock_queue_err_skb, or free via kfree_skb.
 *
 * When passing buffers allocated with this function to sock_queue_err_skb
 * it is necessary to wrap the call with sock_hold/sock_put in order to
 * prevent the socket from being released prior to being enqueued on
 * the sk_error_queue.
 */
4628 4629 4630 4631 4632
struct sk_buff *skb_clone_sk(struct sk_buff *skb)
{
	struct sock *sk = skb->sk;
	struct sk_buff *clone;

4633
	if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648
		return NULL;

	clone = skb_clone(skb, GFP_ATOMIC);
	if (!clone) {
		sock_put(sk);
		return NULL;
	}

	clone->sk = sk;
	clone->destructor = sock_efree;

	return clone;
}
EXPORT_SYMBOL(skb_clone_sk);

4649 4650
static void __skb_complete_tx_timestamp(struct sk_buff *skb,
					struct sock *sk,
4651 4652
					int tstype,
					bool opt_stats)
4653 4654 4655 4656
{
	struct sock_exterr_skb *serr;
	int err;

4657 4658
	BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));

4659 4660 4661 4662
	serr = SKB_EXT_ERR(skb);
	memset(serr, 0, sizeof(*serr));
	serr->ee.ee_errno = ENOMSG;
	serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4663
	serr->ee.ee_info = tstype;
4664
	serr->opt_stats = opt_stats;
4665
	serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4666
	if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4667
		serr->ee.ee_data = skb_shinfo(skb)->tskey;
4668 4669
		if (sk->sk_protocol == IPPROTO_TCP &&
		    sk->sk_type == SOCK_STREAM)
4670 4671
			serr->ee.ee_data -= sk->sk_tskey;
	}
4672

4673
	err = sock_queue_err_skb(sk, skb);
4674

4675 4676 4677
	if (err)
		kfree_skb(skb);
}
4678

4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692
static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
{
	bool ret;

	if (likely(sysctl_tstamp_allow_data || tsonly))
		return true;

	read_lock_bh(&sk->sk_callback_lock);
	ret = sk->sk_socket && sk->sk_socket->file &&
	      file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
	read_unlock_bh(&sk->sk_callback_lock);
	return ret;
}

4693 4694 4695 4696 4697
void skb_complete_tx_timestamp(struct sk_buff *skb,
			       struct skb_shared_hwtstamps *hwtstamps)
{
	struct sock *sk = skb->sk;

4698
	if (!skb_may_tx_timestamp(sk, false))
4699
		goto err;
4700

4701 4702 4703
	/* Take a reference to prevent skb_orphan() from freeing the socket,
	 * but only if the socket refcount is not zero.
	 */
4704
	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4705
		*skb_hwtstamps(skb) = *hwtstamps;
4706
		__skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
4707
		sock_put(sk);
4708
		return;
4709
	}
4710 4711 4712

err:
	kfree_skb(skb);
4713 4714 4715 4716 4717 4718 4719 4720
}
EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);

void __skb_tstamp_tx(struct sk_buff *orig_skb,
		     struct skb_shared_hwtstamps *hwtstamps,
		     struct sock *sk, int tstype)
{
	struct sk_buff *skb;
4721
	bool tsonly, opt_stats = false;
4722

4723 4724 4725
	if (!sk)
		return;

4726 4727 4728 4729
	if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
	    skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
		return;

4730 4731
	tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
	if (!skb_may_tx_timestamp(sk, tsonly))
4732 4733
		return;

4734 4735 4736 4737
	if (tsonly) {
#ifdef CONFIG_INET
		if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
		    sk->sk_protocol == IPPROTO_TCP &&
4738
		    sk->sk_type == SOCK_STREAM) {
4739
			skb = tcp_get_timestamping_opt_stats(sk, orig_skb);
4740 4741
			opt_stats = true;
		} else
4742 4743 4744
#endif
			skb = alloc_skb(0, GFP_ATOMIC);
	} else {
4745
		skb = skb_clone(orig_skb, GFP_ATOMIC);
4746
	}
4747 4748 4749
	if (!skb)
		return;

4750
	if (tsonly) {
4751 4752
		skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
					     SKBTX_ANY_TSTAMP;
4753 4754 4755 4756 4757 4758 4759 4760
		skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
	}

	if (hwtstamps)
		*skb_hwtstamps(skb) = *hwtstamps;
	else
		skb->tstamp = ktime_get_real();

4761
	__skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
4762
}
4763 4764 4765 4766 4767 4768 4769 4770
EXPORT_SYMBOL_GPL(__skb_tstamp_tx);

void skb_tstamp_tx(struct sk_buff *orig_skb,
		   struct skb_shared_hwtstamps *hwtstamps)
{
	return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
			       SCM_TSTAMP_SND);
}
4771 4772
EXPORT_SYMBOL_GPL(skb_tstamp_tx);

4773 4774 4775 4776
void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
{
	struct sock *sk = skb->sk;
	struct sock_exterr_skb *serr;
4777
	int err = 1;
4778 4779 4780 4781 4782 4783 4784 4785 4786

	skb->wifi_acked_valid = 1;
	skb->wifi_acked = acked;

	serr = SKB_EXT_ERR(skb);
	memset(serr, 0, sizeof(*serr));
	serr->ee.ee_errno = ENOMSG;
	serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;

4787 4788 4789
	/* Take a reference to prevent skb_orphan() from freeing the socket,
	 * but only if the socket refcount is not zero.
	 */
4790
	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4791 4792 4793
		err = sock_queue_err_skb(sk, skb);
		sock_put(sk);
	}
4794 4795 4796 4797 4798
	if (err)
		kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);

4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812
/**
 * skb_partial_csum_set - set up and verify partial csum values for packet
 * @skb: the skb to set
 * @start: the number of bytes after skb->data to start checksumming.
 * @off: the offset from start to place the checksum.
 *
 * For untrusted partially-checksummed packets, we need to make sure the values
 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
 *
 * This function checks and sets those values and skb->ip_summed: if this
 * returns false you should drop the packet.
 */
bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
{
4813 4814 4815 4816 4817 4818
	u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
	u32 csum_start = skb_headroom(skb) + (u32)start;

	if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) {
		net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
				     start, off, skb_headroom(skb), skb_headlen(skb));
4819 4820 4821
		return false;
	}
	skb->ip_summed = CHECKSUM_PARTIAL;
4822
	skb->csum_start = csum_start;
4823
	skb->csum_offset = off;
4824
	skb_set_transport_header(skb, start);
4825 4826
	return true;
}
4827
EXPORT_SYMBOL_GPL(skb_partial_csum_set);
4828

4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849
static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
			       unsigned int max)
{
	if (skb_headlen(skb) >= len)
		return 0;

	/* If we need to pullup then pullup to the max, so we
	 * won't need to do it again.
	 */
	if (max > skb->len)
		max = skb->len;

	if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
		return -ENOMEM;

	if (skb_headlen(skb) < len)
		return -EPROTO;

	return 0;
}

4850 4851 4852 4853 4854 4855
#define MAX_TCP_HDR_LEN (15 * 4)

static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
				      typeof(IPPROTO_IP) proto,
				      unsigned int off)
{
4856
	int err;
4857

4858
	switch (proto) {
4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880
	case IPPROTO_TCP:
		err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
					  off + MAX_TCP_HDR_LEN);
		if (!err && !skb_partial_csum_set(skb, off,
						  offsetof(struct tcphdr,
							   check)))
			err = -EPROTO;
		return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;

	case IPPROTO_UDP:
		err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
					  off + sizeof(struct udphdr));
		if (!err && !skb_partial_csum_set(skb, off,
						  offsetof(struct udphdr,
							   check)))
			err = -EPROTO;
		return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
	}

	return ERR_PTR(-EPROTO);
}

4881 4882 4883 4884 4885
/* This value should be large enough to cover a tagged ethernet header plus
 * maximally sized IP and TCP or UDP headers.
 */
#define MAX_IP_HDR_LEN 128

4886
static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
4887 4888 4889
{
	unsigned int off;
	bool fragment;
4890
	__sum16 *csum;
4891 4892 4893 4894 4895 4896 4897 4898 4899 4900
	int err;

	fragment = false;

	err = skb_maybe_pull_tail(skb,
				  sizeof(struct iphdr),
				  MAX_IP_HDR_LEN);
	if (err < 0)
		goto out;

4901
	if (ip_is_fragment(ip_hdr(skb)))
4902 4903 4904 4905 4906 4907 4908 4909 4910
		fragment = true;

	off = ip_hdrlen(skb);

	err = -EPROTO;

	if (fragment)
		goto out;

4911 4912 4913
	csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
	if (IS_ERR(csum))
		return PTR_ERR(csum);
4914

4915 4916 4917 4918 4919
	if (recalculate)
		*csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
					   ip_hdr(skb)->daddr,
					   skb->len - off,
					   ip_hdr(skb)->protocol, 0);
4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941
	err = 0;

out:
	return err;
}

/* This value should be large enough to cover a tagged ethernet header plus
 * an IPv6 header, all options, and a maximal TCP or UDP header.
 */
#define MAX_IPV6_HDR_LEN 256

#define OPT_HDR(type, skb, off) \
	(type *)(skb_network_header(skb) + (off))

static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
{
	int err;
	u8 nexthdr;
	unsigned int off;
	unsigned int len;
	bool fragment;
	bool done;
4942
	__sum16 *csum;
4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019

	fragment = false;
	done = false;

	off = sizeof(struct ipv6hdr);

	err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
	if (err < 0)
		goto out;

	nexthdr = ipv6_hdr(skb)->nexthdr;

	len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
	while (off <= len && !done) {
		switch (nexthdr) {
		case IPPROTO_DSTOPTS:
		case IPPROTO_HOPOPTS:
		case IPPROTO_ROUTING: {
			struct ipv6_opt_hdr *hp;

			err = skb_maybe_pull_tail(skb,
						  off +
						  sizeof(struct ipv6_opt_hdr),
						  MAX_IPV6_HDR_LEN);
			if (err < 0)
				goto out;

			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
			nexthdr = hp->nexthdr;
			off += ipv6_optlen(hp);
			break;
		}
		case IPPROTO_AH: {
			struct ip_auth_hdr *hp;

			err = skb_maybe_pull_tail(skb,
						  off +
						  sizeof(struct ip_auth_hdr),
						  MAX_IPV6_HDR_LEN);
			if (err < 0)
				goto out;

			hp = OPT_HDR(struct ip_auth_hdr, skb, off);
			nexthdr = hp->nexthdr;
			off += ipv6_authlen(hp);
			break;
		}
		case IPPROTO_FRAGMENT: {
			struct frag_hdr *hp;

			err = skb_maybe_pull_tail(skb,
						  off +
						  sizeof(struct frag_hdr),
						  MAX_IPV6_HDR_LEN);
			if (err < 0)
				goto out;

			hp = OPT_HDR(struct frag_hdr, skb, off);

			if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
				fragment = true;

			nexthdr = hp->nexthdr;
			off += sizeof(struct frag_hdr);
			break;
		}
		default:
			done = true;
			break;
		}
	}

	err = -EPROTO;

	if (!done || fragment)
		goto out;

5020 5021 5022
	csum = skb_checksum_setup_ip(skb, nexthdr, off);
	if (IS_ERR(csum))
		return PTR_ERR(csum);
5023

5024 5025 5026 5027
	if (recalculate)
		*csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
					 &ipv6_hdr(skb)->daddr,
					 skb->len - off, nexthdr, 0);
5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044
	err = 0;

out:
	return err;
}

/**
 * skb_checksum_setup - set up partial checksum offset
 * @skb: the skb to set up
 * @recalculate: if true the pseudo-header checksum will be recalculated
 */
int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
{
	int err;

	switch (skb->protocol) {
	case htons(ETH_P_IP):
5045
		err = skb_checksum_setup_ipv4(skb, recalculate);
5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060
		break;

	case htons(ETH_P_IPV6):
		err = skb_checksum_setup_ipv6(skb, recalculate);
		break;

	default:
		err = -EPROTO;
		break;
	}

	return err;
}
EXPORT_SYMBOL(skb_checksum_setup);

5061 5062 5063 5064 5065 5066 5067 5068 5069 5070
/**
 * skb_checksum_maybe_trim - maybe trims the given skb
 * @skb: the skb to check
 * @transport_len: the data length beyond the network header
 *
 * Checks whether the given skb has data beyond the given transport length.
 * If so, returns a cloned skb trimmed to this transport length.
 * Otherwise returns the provided skb. Returns NULL in error cases
 * (e.g. transport_len exceeds skb length or out-of-memory).
 *
5071 5072
 * Caller needs to set the skb transport header and free any returned skb if it
 * differs from the provided skb.
5073 5074 5075 5076 5077 5078 5079 5080
 */
static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
					       unsigned int transport_len)
{
	struct sk_buff *skb_chk;
	unsigned int len = skb_transport_offset(skb) + transport_len;
	int ret;

5081
	if (skb->len < len)
5082
		return NULL;
5083
	else if (skb->len == len)
5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110
		return skb;

	skb_chk = skb_clone(skb, GFP_ATOMIC);
	if (!skb_chk)
		return NULL;

	ret = pskb_trim_rcsum(skb_chk, len);
	if (ret) {
		kfree_skb(skb_chk);
		return NULL;
	}

	return skb_chk;
}

/**
 * skb_checksum_trimmed - validate checksum of an skb
 * @skb: the skb to check
 * @transport_len: the data length beyond the network header
 * @skb_chkf: checksum function to use
 *
 * Applies the given checksum function skb_chkf to the provided skb.
 * Returns a checked and maybe trimmed skb. Returns NULL on error.
 *
 * If the skb has data beyond the given transport length, then a
 * trimmed & cloned skb is checked and returned.
 *
5111 5112
 * Caller needs to set the skb transport header and free any returned skb if it
 * differs from the provided skb.
5113 5114 5115 5116 5117 5118 5119
 */
struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
				     unsigned int transport_len,
				     __sum16(*skb_chkf)(struct sk_buff *skb))
{
	struct sk_buff *skb_chk;
	unsigned int offset = skb_transport_offset(skb);
5120
	__sum16 ret;
5121 5122 5123

	skb_chk = skb_checksum_maybe_trim(skb, transport_len);
	if (!skb_chk)
5124
		goto err;
5125

5126 5127
	if (!pskb_may_pull(skb_chk, offset))
		goto err;
5128

5129
	skb_pull_rcsum(skb_chk, offset);
5130
	ret = skb_chkf(skb_chk);
5131
	skb_push_rcsum(skb_chk, offset);
5132

5133 5134
	if (ret)
		goto err;
5135 5136

	return skb_chk;
5137 5138 5139 5140 5141 5142 5143

err:
	if (skb_chk && skb_chk != skb)
		kfree_skb(skb_chk);

	return NULL;

5144 5145 5146
}
EXPORT_SYMBOL(skb_checksum_trimmed);

5147 5148
void __skb_warn_lro_forwarding(const struct sk_buff *skb)
{
5149 5150
	net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
			     skb->dev->name);
5151 5152
}
EXPORT_SYMBOL(__skb_warn_lro_forwarding);
5153 5154 5155

void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
{
E
Eric Dumazet 已提交
5156 5157
	if (head_stolen) {
		skb_release_head_state(skb);
5158
		kmem_cache_free(skbuff_head_cache, skb);
E
Eric Dumazet 已提交
5159
	} else {
5160
		__kfree_skb(skb);
E
Eric Dumazet 已提交
5161
	}
5162 5163 5164 5165 5166 5167 5168 5169
}
EXPORT_SYMBOL(kfree_skb_partial);

/**
 * skb_try_coalesce - try to merge skb to prior one
 * @to: prior buffer
 * @from: buffer to add
 * @fragstolen: pointer to boolean
5170
 * @delta_truesize: how much more was allocated than was requested
5171 5172 5173 5174
 */
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
		      bool *fragstolen, int *delta_truesize)
{
5175
	struct skb_shared_info *to_shinfo, *from_shinfo;
5176 5177 5178 5179 5180 5181 5182 5183
	int i, delta, len = from->len;

	*fragstolen = false;

	if (skb_cloned(to))
		return false;

	if (len <= skb_tailroom(to)) {
5184 5185
		if (len)
			BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
5186 5187 5188 5189
		*delta_truesize = 0;
		return true;
	}

5190 5191 5192
	to_shinfo = skb_shinfo(to);
	from_shinfo = skb_shinfo(from);
	if (to_shinfo->frag_list || from_shinfo->frag_list)
5193
		return false;
5194 5195
	if (skb_zcopy(to) || skb_zcopy(from))
		return false;
5196 5197 5198 5199 5200

	if (skb_headlen(from) != 0) {
		struct page *page;
		unsigned int offset;

5201 5202
		if (to_shinfo->nr_frags +
		    from_shinfo->nr_frags >= MAX_SKB_FRAGS)
5203 5204 5205 5206 5207 5208 5209 5210 5211 5212
			return false;

		if (skb_head_is_locked(from))
			return false;

		delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));

		page = virt_to_head_page(from->head);
		offset = from->data - (unsigned char *)page_address(page);

5213
		skb_fill_page_desc(to, to_shinfo->nr_frags,
5214 5215 5216
				   page, offset, skb_headlen(from));
		*fragstolen = true;
	} else {
5217 5218
		if (to_shinfo->nr_frags +
		    from_shinfo->nr_frags > MAX_SKB_FRAGS)
5219 5220
			return false;

5221
		delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
5222 5223 5224 5225
	}

	WARN_ON_ONCE(delta < len);

5226 5227 5228 5229
	memcpy(to_shinfo->frags + to_shinfo->nr_frags,
	       from_shinfo->frags,
	       from_shinfo->nr_frags * sizeof(skb_frag_t));
	to_shinfo->nr_frags += from_shinfo->nr_frags;
5230 5231

	if (!skb_cloned(from))
5232
		from_shinfo->nr_frags = 0;
5233

5234 5235 5236
	/* if the skb is not cloned this does nothing
	 * since we set nr_frags to 0.
	 */
5237 5238
	for (i = 0; i < from_shinfo->nr_frags; i++)
		__skb_frag_ref(&from_shinfo->frags[i]);
5239 5240 5241 5242 5243 5244 5245 5246 5247

	to->truesize += delta;
	to->len += len;
	to->data_len += len;

	*delta_truesize = delta;
	return true;
}
EXPORT_SYMBOL(skb_try_coalesce);
5248 5249

/**
5250
 * skb_scrub_packet - scrub an skb
5251 5252
 *
 * @skb: buffer to clean
5253 5254 5255 5256 5257 5258 5259 5260
 * @xnet: packet is crossing netns
 *
 * skb_scrub_packet can be used after encapsulating or decapsulting a packet
 * into/from a tunnel. Some information have to be cleared during these
 * operations.
 * skb_scrub_packet can also be used to clean a skb before injecting it in
 * another namespace (@xnet == true). We have to clear all information in the
 * skb that could impact namespace isolation.
5261
 */
5262
void skb_scrub_packet(struct sk_buff *skb, bool xnet)
5263 5264 5265
{
	skb->pkt_type = PACKET_HOST;
	skb->skb_iif = 0;
5266
	skb->ignore_df = 0;
5267
	skb_dst_drop(skb);
5268
	skb_ext_reset(skb);
5269
	nf_reset_ct(skb);
5270
	nf_reset_trace(skb);
5271

5272 5273
#ifdef CONFIG_NET_SWITCHDEV
	skb->offload_fwd_mark = 0;
5274
	skb->offload_l3_fwd_mark = 0;
5275 5276
#endif

5277 5278 5279
	if (!xnet)
		return;

5280
	ipvs_reset(skb);
5281
	skb->mark = 0;
5282
	skb->tstamp = 0;
5283 5284
}
EXPORT_SYMBOL_GPL(skb_scrub_packet);
5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295

/**
 * skb_gso_transport_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_transport_seglen is used to determine the real size of the
 * individual segments, including Layer4 headers (TCP/UDP).
 *
 * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
 */
5296
static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
5297 5298
{
	const struct skb_shared_info *shinfo = skb_shinfo(skb);
5299
	unsigned int thlen = 0;
5300

5301 5302 5303
	if (skb->encapsulation) {
		thlen = skb_inner_transport_header(skb) -
			skb_transport_header(skb);
5304

5305 5306 5307 5308
		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
			thlen += inner_tcp_hdrlen(skb);
	} else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
		thlen = tcp_hdrlen(skb);
5309
	} else if (unlikely(skb_is_gso_sctp(skb))) {
5310
		thlen = sizeof(struct sctphdr);
W
Willem de Bruijn 已提交
5311 5312
	} else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
		thlen = sizeof(struct udphdr);
5313
	}
5314 5315 5316 5317
	/* UFO sets gso_size to the size of the fragmentation
	 * payload, i.e. the size of the L4 (UDP) header is already
	 * accounted for.
	 */
5318
	return thlen + shinfo->gso_size;
5319
}
5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353

/**
 * skb_gso_network_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_network_seglen is used to determine the real size of the
 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
 *
 * The MAC/L2 header is not accounted for.
 */
static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
{
	unsigned int hdr_len = skb_transport_header(skb) -
			       skb_network_header(skb);

	return hdr_len + skb_gso_transport_seglen(skb);
}

/**
 * skb_gso_mac_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_mac_seglen is used to determine the real size of the
 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
 * headers (TCP/UDP).
 */
static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
{
	unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);

	return hdr_len + skb_gso_transport_seglen(skb);
}
5354

5355
/**
5356
 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
5357
 *
5358 5359
 * There are a couple of instances where we have a GSO skb, and we
 * want to determine what size it would be after it is segmented.
5360
 *
5361 5362 5363 5364 5365 5366
 * We might want to check:
 * -    L3+L4+payload size (e.g. IP forwarding)
 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
 *
 * This is a helper to do that correctly considering GSO_BY_FRAGS.
 *
5367 5368
 * @skb: GSO skb
 *
5369 5370 5371 5372 5373 5374
 * @seg_len: The segmented length (from skb_gso_*_seglen). In the
 *           GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
 *
 * @max_len: The maximum permissible length.
 *
 * Returns true if the segmented length <= max length.
5375
 */
5376 5377 5378
static inline bool skb_gso_size_check(const struct sk_buff *skb,
				      unsigned int seg_len,
				      unsigned int max_len) {
5379 5380 5381 5382
	const struct skb_shared_info *shinfo = skb_shinfo(skb);
	const struct sk_buff *iter;

	if (shinfo->gso_size != GSO_BY_FRAGS)
5383
		return seg_len <= max_len;
5384 5385

	/* Undo this so we can re-use header sizes */
5386
	seg_len -= GSO_BY_FRAGS;
5387 5388

	skb_walk_frags(skb, iter) {
5389
		if (seg_len + skb_headlen(iter) > max_len)
5390 5391 5392 5393 5394
			return false;
	}

	return true;
}
5395 5396

/**
5397
 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
5398 5399 5400 5401
 *
 * @skb: GSO skb
 * @mtu: MTU to validate against
 *
5402 5403 5404
 * skb_gso_validate_network_len validates if a given skb will fit a
 * wanted MTU once split. It considers L3 headers, L4 headers, and the
 * payload.
5405
 */
5406
bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
5407 5408 5409
{
	return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
}
5410
EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
5411

5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426
/**
 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
 *
 * @skb: GSO skb
 * @len: length to validate against
 *
 * skb_gso_validate_mac_len validates if a given skb will fit a wanted
 * length once split, including L2, L3 and L4 headers and the payload.
 */
bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
{
	return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
}
EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);

5427 5428
static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
{
5429 5430
	int mac_len, meta_len;
	void *meta;
5431

5432 5433 5434 5435 5436
	if (skb_cow(skb, skb_headroom(skb)) < 0) {
		kfree_skb(skb);
		return NULL;
	}

5437
	mac_len = skb->data - skb_mac_header(skb);
5438 5439 5440 5441
	if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
		memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
			mac_len - VLAN_HLEN - ETH_TLEN);
	}
5442 5443 5444 5445 5446 5447 5448

	meta_len = skb_metadata_len(skb);
	if (meta_len) {
		meta = skb_metadata_end(skb) - meta_len;
		memmove(meta + VLAN_HLEN, meta, meta_len);
	}

5449 5450 5451 5452 5453 5454 5455 5456 5457
	skb->mac_header += VLAN_HLEN;
	return skb;
}

struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
{
	struct vlan_hdr *vhdr;
	u16 vlan_tci;

5458
	if (unlikely(skb_vlan_tag_present(skb))) {
5459 5460 5461 5462 5463 5464 5465
		/* vlan_tci is already set-up so leave this for another time */
		return skb;
	}

	skb = skb_share_check(skb, GFP_ATOMIC);
	if (unlikely(!skb))
		goto err_free;
5466 5467
	/* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */
	if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short))))
5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481
		goto err_free;

	vhdr = (struct vlan_hdr *)skb->data;
	vlan_tci = ntohs(vhdr->h_vlan_TCI);
	__vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);

	skb_pull_rcsum(skb, VLAN_HLEN);
	vlan_set_encap_proto(skb, vhdr);

	skb = skb_reorder_vlan_header(skb);
	if (unlikely(!skb))
		goto err_free;

	skb_reset_network_header(skb);
5482 5483
	if (!skb_transport_header_was_set(skb))
		skb_reset_transport_header(skb);
5484 5485 5486 5487 5488 5489 5490 5491 5492
	skb_reset_mac_len(skb);

	return skb;

err_free:
	kfree_skb(skb);
	return NULL;
}
EXPORT_SYMBOL(skb_vlan_untag);
5493

5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505
int skb_ensure_writable(struct sk_buff *skb, int write_len)
{
	if (!pskb_may_pull(skb, write_len))
		return -ENOMEM;

	if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
		return 0;

	return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
}
EXPORT_SYMBOL(skb_ensure_writable);

5506 5507 5508 5509
/* remove VLAN header from packet and update csum accordingly.
 * expects a non skb_vlan_tag_present skb with a vlan tag payload
 */
int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
5510 5511
{
	struct vlan_hdr *vhdr;
5512
	int offset = skb->data - skb_mac_header(skb);
5513 5514
	int err;

5515 5516 5517 5518 5519 5520
	if (WARN_ONCE(offset,
		      "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
		      offset)) {
		return -EINVAL;
	}

5521 5522
	err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
	if (unlikely(err))
5523
		return err;
5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542

	skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);

	vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
	*vlan_tci = ntohs(vhdr->h_vlan_TCI);

	memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
	__skb_pull(skb, VLAN_HLEN);

	vlan_set_encap_proto(skb, vhdr);
	skb->mac_header += VLAN_HLEN;

	if (skb_network_offset(skb) < ETH_HLEN)
		skb_set_network_header(skb, ETH_HLEN);

	skb_reset_mac_len(skb);

	return err;
}
5543
EXPORT_SYMBOL(__skb_vlan_pop);
5544

5545 5546 5547
/* Pop a vlan tag either from hwaccel or from payload.
 * Expects skb->data at mac header.
 */
5548 5549 5550 5551 5552 5553
int skb_vlan_pop(struct sk_buff *skb)
{
	u16 vlan_tci;
	__be16 vlan_proto;
	int err;

5554
	if (likely(skb_vlan_tag_present(skb))) {
5555
		__vlan_hwaccel_clear_tag(skb);
5556
	} else {
5557
		if (unlikely(!eth_type_vlan(skb->protocol)))
5558 5559 5560 5561 5562 5563 5564
			return 0;

		err = __skb_vlan_pop(skb, &vlan_tci);
		if (err)
			return err;
	}
	/* move next vlan tag to hw accel tag */
5565
	if (likely(!eth_type_vlan(skb->protocol)))
5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577
		return 0;

	vlan_proto = skb->protocol;
	err = __skb_vlan_pop(skb, &vlan_tci);
	if (unlikely(err))
		return err;

	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
	return 0;
}
EXPORT_SYMBOL(skb_vlan_pop);

5578 5579 5580
/* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
 * Expects skb->data at mac header.
 */
5581 5582
int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
{
5583
	if (skb_vlan_tag_present(skb)) {
5584
		int offset = skb->data - skb_mac_header(skb);
5585 5586
		int err;

5587 5588 5589 5590 5591 5592
		if (WARN_ONCE(offset,
			      "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
			      offset)) {
			return -EINVAL;
		}

5593
		err = __vlan_insert_tag(skb, skb->vlan_proto,
5594
					skb_vlan_tag_get(skb));
5595
		if (err)
5596
			return err;
5597

5598 5599 5600
		skb->protocol = skb->vlan_proto;
		skb->mac_len += VLAN_HLEN;

5601
		skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5602 5603 5604 5605 5606 5607
	}
	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
	return 0;
}
EXPORT_SYMBOL(skb_vlan_push);

5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674
/**
 * skb_eth_pop() - Drop the Ethernet header at the head of a packet
 *
 * @skb: Socket buffer to modify
 *
 * Drop the Ethernet header of @skb.
 *
 * Expects that skb->data points to the mac header and that no VLAN tags are
 * present.
 *
 * Returns 0 on success, -errno otherwise.
 */
int skb_eth_pop(struct sk_buff *skb)
{
	if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
	    skb_network_offset(skb) < ETH_HLEN)
		return -EPROTO;

	skb_pull_rcsum(skb, ETH_HLEN);
	skb_reset_mac_header(skb);
	skb_reset_mac_len(skb);

	return 0;
}
EXPORT_SYMBOL(skb_eth_pop);

/**
 * skb_eth_push() - Add a new Ethernet header at the head of a packet
 *
 * @skb: Socket buffer to modify
 * @dst: Destination MAC address of the new header
 * @src: Source MAC address of the new header
 *
 * Prepend @skb with a new Ethernet header.
 *
 * Expects that skb->data points to the mac header, which must be empty.
 *
 * Returns 0 on success, -errno otherwise.
 */
int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
		 const unsigned char *src)
{
	struct ethhdr *eth;
	int err;

	if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
		return -EPROTO;

	err = skb_cow_head(skb, sizeof(*eth));
	if (err < 0)
		return err;

	skb_push(skb, sizeof(*eth));
	skb_reset_mac_header(skb);
	skb_reset_mac_len(skb);

	eth = eth_hdr(skb);
	ether_addr_copy(eth->h_dest, dst);
	ether_addr_copy(eth->h_source, src);
	eth->h_proto = skb->protocol;

	skb_postpush_rcsum(skb, eth, sizeof(*eth));

	return 0;
}
EXPORT_SYMBOL(skb_eth_push);

5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688
/* Update the ethertype of hdr and the skb csum value if required. */
static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
			     __be16 ethertype)
{
	if (skb->ip_summed == CHECKSUM_COMPLETE) {
		__be16 diff[] = { ~hdr->h_proto, ethertype };

		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
	}

	hdr->h_proto = ethertype;
}

/**
5689 5690
 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
 *                   the packet
5691 5692 5693 5694
 *
 * @skb: buffer
 * @mpls_lse: MPLS label stack entry to push
 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
5695
 * @mac_len: length of the MAC header
5696 5697
 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
 *            ethernet
5698 5699 5700 5701 5702
 *
 * Expects skb->data at mac header.
 *
 * Returns 0 on success, -errno otherwise.
 */
5703
int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
5704
		  int mac_len, bool ethernet)
5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720
{
	struct mpls_shim_hdr *lse;
	int err;

	if (unlikely(!eth_p_mpls(mpls_proto)))
		return -EINVAL;

	/* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
	if (skb->encapsulation)
		return -EINVAL;

	err = skb_cow_head(skb, MPLS_HLEN);
	if (unlikely(err))
		return err;

	if (!skb->inner_protocol) {
5721
		skb_set_inner_network_header(skb, skb_network_offset(skb));
5722 5723 5724 5725 5726
		skb_set_inner_protocol(skb, skb->protocol);
	}

	skb_push(skb, MPLS_HLEN);
	memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
5727
		mac_len);
5728
	skb_reset_mac_header(skb);
5729
	skb_set_network_header(skb, mac_len);
5730
	skb_reset_mac_len(skb);
5731 5732 5733 5734 5735

	lse = mpls_hdr(skb);
	lse->label_stack_entry = mpls_lse;
	skb_postpush_rcsum(skb, lse, MPLS_HLEN);

5736
	if (ethernet && mac_len >= ETH_HLEN)
5737 5738 5739 5740 5741 5742 5743
		skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
	skb->protocol = mpls_proto;

	return 0;
}
EXPORT_SYMBOL_GPL(skb_mpls_push);

5744 5745 5746 5747 5748
/**
 * skb_mpls_pop() - pop the outermost MPLS header
 *
 * @skb: buffer
 * @next_proto: ethertype of header after popped MPLS header
5749
 * @mac_len: length of the MAC header
5750
 * @ethernet: flag to indicate if the packet is ethernet
5751 5752 5753 5754 5755
 *
 * Expects skb->data at mac header.
 *
 * Returns 0 on success, -errno otherwise.
 */
5756 5757
int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
		 bool ethernet)
5758 5759 5760 5761
{
	int err;

	if (unlikely(!eth_p_mpls(skb->protocol)))
5762
		return 0;
5763

5764
	err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
5765 5766 5767 5768 5769
	if (unlikely(err))
		return err;

	skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
	memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
5770
		mac_len);
5771 5772 5773

	__skb_pull(skb, MPLS_HLEN);
	skb_reset_mac_header(skb);
5774
	skb_set_network_header(skb, mac_len);
5775

5776
	if (ethernet && mac_len >= ETH_HLEN) {
5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788
		struct ethhdr *hdr;

		/* use mpls_hdr() to get ethertype to account for VLANs. */
		hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
		skb_mod_eth_type(skb, hdr, next_proto);
	}
	skb->protocol = next_proto;

	return 0;
}
EXPORT_SYMBOL_GPL(skb_mpls_pop);

5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821
/**
 * skb_mpls_update_lse() - modify outermost MPLS header and update csum
 *
 * @skb: buffer
 * @mpls_lse: new MPLS label stack entry to update to
 *
 * Expects skb->data at mac header.
 *
 * Returns 0 on success, -errno otherwise.
 */
int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
{
	int err;

	if (unlikely(!eth_p_mpls(skb->protocol)))
		return -EINVAL;

	err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
	if (unlikely(err))
		return err;

	if (skb->ip_summed == CHECKSUM_COMPLETE) {
		__be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };

		skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
	}

	mpls_hdr(skb)->label_stack_entry = mpls_lse;

	return 0;
}
EXPORT_SYMBOL_GPL(skb_mpls_update_lse);

5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838
/**
 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
 *
 * @skb: buffer
 *
 * Expects skb->data at mac header.
 *
 * Returns 0 on success, -errno otherwise.
 */
int skb_mpls_dec_ttl(struct sk_buff *skb)
{
	u32 lse;
	u8 ttl;

	if (unlikely(!eth_p_mpls(skb->protocol)))
		return -EINVAL;

5839 5840 5841
	if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
		return -ENOMEM;

5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853
	lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
	ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
	if (!--ttl)
		return -EINVAL;

	lse &= ~MPLS_LS_TTL_MASK;
	lse |= ttl << MPLS_LS_TTL_SHIFT;

	return skb_mpls_update_lse(skb, cpu_to_be32(lse));
}
EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);

5854 5855 5856
/**
 * alloc_skb_with_frags - allocate skb with page frags
 *
5857 5858 5859 5860 5861
 * @header_len: size of linear part
 * @data_len: needed length in frags
 * @max_page_order: max page order desired.
 * @errcode: pointer to error code if any
 * @gfp_mask: allocation mask
5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884
 *
 * This can be used to allocate a paged skb, given a maximal order for frags.
 */
struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
				     unsigned long data_len,
				     int max_page_order,
				     int *errcode,
				     gfp_t gfp_mask)
{
	int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	unsigned long chunk;
	struct sk_buff *skb;
	struct page *page;
	int i;

	*errcode = -EMSGSIZE;
	/* Note this test could be relaxed, if we succeed to allocate
	 * high order pages...
	 */
	if (npages > MAX_SKB_FRAGS)
		return NULL;

	*errcode = -ENOBUFS;
5885
	skb = alloc_skb(header_len, gfp_mask);
5886 5887 5888 5889 5890 5891 5892 5893 5894 5895
	if (!skb)
		return NULL;

	skb->truesize += npages << PAGE_SHIFT;

	for (i = 0; npages > 0; i++) {
		int order = max_page_order;

		while (order) {
			if (npages >= 1 << order) {
5896
				page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
5897
						   __GFP_COMP |
5898
						   __GFP_NOWARN,
5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924
						   order);
				if (page)
					goto fill_page;
				/* Do not retry other high order allocations */
				order = 1;
				max_page_order = 0;
			}
			order--;
		}
		page = alloc_page(gfp_mask);
		if (!page)
			goto failure;
fill_page:
		chunk = min_t(unsigned long, data_len,
			      PAGE_SIZE << order);
		skb_fill_page_desc(skb, i, page, 0, chunk);
		data_len -= chunk;
		npages -= 1 << order;
	}
	return skb;

failure:
	kfree_skb(skb);
	return NULL;
}
EXPORT_SYMBOL(alloc_skb_with_frags);
5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071

/* carve out the first off bytes from skb when off < headlen */
static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
				    const int headlen, gfp_t gfp_mask)
{
	int i;
	int size = skb_end_offset(skb);
	int new_hlen = headlen - off;
	u8 *data;

	size = SKB_DATA_ALIGN(size);

	if (skb_pfmemalloc(skb))
		gfp_mask |= __GFP_MEMALLOC;
	data = kmalloc_reserve(size +
			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
			       gfp_mask, NUMA_NO_NODE, NULL);
	if (!data)
		return -ENOMEM;

	size = SKB_WITH_OVERHEAD(ksize(data));

	/* Copy real data, and all frags */
	skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
	skb->len -= off;

	memcpy((struct skb_shared_info *)(data + size),
	       skb_shinfo(skb),
	       offsetof(struct skb_shared_info,
			frags[skb_shinfo(skb)->nr_frags]));
	if (skb_cloned(skb)) {
		/* drop the old head gracefully */
		if (skb_orphan_frags(skb, gfp_mask)) {
			kfree(data);
			return -ENOMEM;
		}
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
			skb_frag_ref(skb, i);
		if (skb_has_frag_list(skb))
			skb_clone_fraglist(skb);
		skb_release_data(skb);
	} else {
		/* we can reuse existing recount- all we did was
		 * relocate values
		 */
		skb_free_head(skb);
	}

	skb->head = data;
	skb->data = data;
	skb->head_frag = 0;
#ifdef NET_SKBUFF_DATA_USES_OFFSET
	skb->end = size;
#else
	skb->end = skb->head + size;
#endif
	skb_set_tail_pointer(skb, skb_headlen(skb));
	skb_headers_offset_update(skb, 0);
	skb->cloned = 0;
	skb->hdr_len = 0;
	skb->nohdr = 0;
	atomic_set(&skb_shinfo(skb)->dataref, 1);

	return 0;
}

static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);

/* carve out the first eat bytes from skb's frag_list. May recurse into
 * pskb_carve()
 */
static int pskb_carve_frag_list(struct sk_buff *skb,
				struct skb_shared_info *shinfo, int eat,
				gfp_t gfp_mask)
{
	struct sk_buff *list = shinfo->frag_list;
	struct sk_buff *clone = NULL;
	struct sk_buff *insp = NULL;

	do {
		if (!list) {
			pr_err("Not enough bytes to eat. Want %d\n", eat);
			return -EFAULT;
		}
		if (list->len <= eat) {
			/* Eaten as whole. */
			eat -= list->len;
			list = list->next;
			insp = list;
		} else {
			/* Eaten partially. */
			if (skb_shared(list)) {
				clone = skb_clone(list, gfp_mask);
				if (!clone)
					return -ENOMEM;
				insp = list->next;
				list = clone;
			} else {
				/* This may be pulled without problems. */
				insp = list;
			}
			if (pskb_carve(list, eat, gfp_mask) < 0) {
				kfree_skb(clone);
				return -ENOMEM;
			}
			break;
		}
	} while (eat);

	/* Free pulled out fragments. */
	while ((list = shinfo->frag_list) != insp) {
		shinfo->frag_list = list->next;
		kfree_skb(list);
	}
	/* And insert new clone at head. */
	if (clone) {
		clone->next = list;
		shinfo->frag_list = clone;
	}
	return 0;
}

/* carve off first len bytes from skb. Split line (off) is in the
 * non-linear part of skb
 */
static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
				       int pos, gfp_t gfp_mask)
{
	int i, k = 0;
	int size = skb_end_offset(skb);
	u8 *data;
	const int nfrags = skb_shinfo(skb)->nr_frags;
	struct skb_shared_info *shinfo;

	size = SKB_DATA_ALIGN(size);

	if (skb_pfmemalloc(skb))
		gfp_mask |= __GFP_MEMALLOC;
	data = kmalloc_reserve(size +
			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
			       gfp_mask, NUMA_NO_NODE, NULL);
	if (!data)
		return -ENOMEM;

	size = SKB_WITH_OVERHEAD(ksize(data));

	memcpy((struct skb_shared_info *)(data + size),
6072
	       skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092
	if (skb_orphan_frags(skb, gfp_mask)) {
		kfree(data);
		return -ENOMEM;
	}
	shinfo = (struct skb_shared_info *)(data + size);
	for (i = 0; i < nfrags; i++) {
		int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);

		if (pos + fsize > off) {
			shinfo->frags[k] = skb_shinfo(skb)->frags[i];

			if (pos < off) {
				/* Split frag.
				 * We have two variants in this case:
				 * 1. Move all the frag to the second
				 *    part, if it is possible. F.e.
				 *    this approach is mandatory for TUX,
				 *    where splitting is expensive.
				 * 2. Split is accurately. We make this.
				 */
6093
				skb_frag_off_add(&shinfo->frags[0], off - pos);
6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104
				skb_frag_size_sub(&shinfo->frags[0], off - pos);
			}
			skb_frag_ref(skb, i);
			k++;
		}
		pos += fsize;
	}
	shinfo->nr_frags = k;
	if (skb_has_frag_list(skb))
		skb_clone_fraglist(skb);

6105 6106 6107 6108 6109 6110 6111
	/* split line is in frag list */
	if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) {
		/* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
		if (skb_has_frag_list(skb))
			kfree_skb_list(skb_shinfo(skb)->frag_list);
		kfree(data);
		return -ENOMEM;
6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163
	}
	skb_release_data(skb);

	skb->head = data;
	skb->head_frag = 0;
	skb->data = data;
#ifdef NET_SKBUFF_DATA_USES_OFFSET
	skb->end = size;
#else
	skb->end = skb->head + size;
#endif
	skb_reset_tail_pointer(skb);
	skb_headers_offset_update(skb, 0);
	skb->cloned   = 0;
	skb->hdr_len  = 0;
	skb->nohdr    = 0;
	skb->len -= off;
	skb->data_len = skb->len;
	atomic_set(&skb_shinfo(skb)->dataref, 1);
	return 0;
}

/* remove len bytes from the beginning of the skb */
static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
{
	int headlen = skb_headlen(skb);

	if (len < headlen)
		return pskb_carve_inside_header(skb, len, headlen, gfp);
	else
		return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
}

/* Extract to_copy bytes starting at off from skb, and return this in
 * a new skb
 */
struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
			     int to_copy, gfp_t gfp)
{
	struct sk_buff  *clone = skb_clone(skb, gfp);

	if (!clone)
		return NULL;

	if (pskb_carve(clone, off, gfp) < 0 ||
	    pskb_trim(clone, to_copy)) {
		kfree_skb(clone);
		return NULL;
	}
	return clone;
}
EXPORT_SYMBOL(pskb_extract);
6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178

/**
 * skb_condense - try to get rid of fragments/frag_list if possible
 * @skb: buffer
 *
 * Can be used to save memory before skb is added to a busy queue.
 * If packet has bytes in frags and enough tail room in skb->head,
 * pull all of them, so that we can free the frags right now and adjust
 * truesize.
 * Notes:
 *	We do not reallocate skb->head thus can not fail.
 *	Caller must re-evaluate skb->truesize if needed.
 */
void skb_condense(struct sk_buff *skb)
{
6179 6180 6181 6182
	if (skb->data_len) {
		if (skb->data_len > skb->end - skb->tail ||
		    skb_cloned(skb))
			return;
6183

6184 6185 6186 6187 6188 6189 6190 6191 6192
		/* Nice, we can free page frag(s) right now */
		__pskb_pull_tail(skb, skb->data_len);
	}
	/* At this point, skb->truesize might be over estimated,
	 * because skb had a fragment, and fragments do not tell
	 * their truesize.
	 * When we pulled its content into skb->head, fragment
	 * was freed, but __pskb_pull_tail() could not possibly
	 * adjust skb->truesize, not knowing the frag truesize.
6193 6194 6195
	 */
	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
}
6196 6197 6198 6199 6200 6201 6202

#ifdef CONFIG_SKB_EXTENSIONS
static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
{
	return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
}

6203 6204 6205
/**
 * __skb_ext_alloc - allocate a new skb extensions storage
 *
6206 6207
 * @flags: See kmalloc().
 *
6208 6209 6210 6211
 * Returns the newly allocated pointer. The pointer can later attached to a
 * skb via __skb_ext_set().
 * Note: caller must handle the skb_ext as an opaque data.
 */
6212
struct skb_ext *__skb_ext_alloc(gfp_t flags)
6213
{
6214
	struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
6215 6216 6217 6218 6219 6220 6221 6222 6223

	if (new) {
		memset(new->offset, 0, sizeof(new->offset));
		refcount_set(&new->refcnt, 1);
	}

	return new;
}

6224 6225
static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
					 unsigned int old_active)
6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238
{
	struct skb_ext *new;

	if (refcount_read(&old->refcnt) == 1)
		return old;

	new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
	if (!new)
		return NULL;

	memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
	refcount_set(&new->refcnt, 1);

6239 6240 6241 6242 6243 6244 6245 6246 6247
#ifdef CONFIG_XFRM
	if (old_active & (1 << SKB_EXT_SEC_PATH)) {
		struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
		unsigned int i;

		for (i = 0; i < sp->len; i++)
			xfrm_state_hold(sp->xvec[i]);
	}
#endif
6248 6249 6250 6251
	__skb_ext_put(old);
	return new;
}

6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275
/**
 * __skb_ext_set - attach the specified extension storage to this skb
 * @skb: buffer
 * @id: extension id
 * @ext: extension storage previously allocated via __skb_ext_alloc()
 *
 * Existing extensions, if any, are cleared.
 *
 * Returns the pointer to the extension.
 */
void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
		    struct skb_ext *ext)
{
	unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);

	skb_ext_put(skb);
	newlen = newoff + skb_ext_type_len[id];
	ext->chunks = newlen;
	ext->offset[id] = newoff;
	skb->extensions = ext;
	skb->active_extensions = 1 << id;
	return skb_ext_get_ptr(ext, id);
}

6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297
/**
 * skb_ext_add - allocate space for given extension, COW if needed
 * @skb: buffer
 * @id: extension to allocate space for
 *
 * Allocates enough space for the given extension.
 * If the extension is already present, a pointer to that extension
 * is returned.
 *
 * If the skb was cloned, COW applies and the returned memory can be
 * modified without changing the extension space of clones buffers.
 *
 * Returns pointer to the extension or NULL on allocation failure.
 */
void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
{
	struct skb_ext *new, *old = NULL;
	unsigned int newlen, newoff;

	if (skb->active_extensions) {
		old = skb->extensions;

6298
		new = skb_ext_maybe_cow(old, skb->active_extensions);
6299 6300 6301
		if (!new)
			return NULL;

6302
		if (__skb_ext_exist(new, id))
6303 6304
			goto set_active;

6305
		newoff = new->chunks;
6306 6307 6308
	} else {
		newoff = SKB_EXT_CHUNKSIZEOF(*new);

6309
		new = __skb_ext_alloc(GFP_ATOMIC);
6310 6311 6312 6313 6314 6315 6316 6317
		if (!new)
			return NULL;
	}

	newlen = newoff + skb_ext_type_len[id];
	new->chunks = newlen;
	new->offset[id] = newoff;
set_active:
6318
	skb->extensions = new;
6319 6320 6321 6322 6323
	skb->active_extensions |= 1 << id;
	return skb_ext_get_ptr(new, id);
}
EXPORT_SYMBOL(skb_ext_add);

6324 6325 6326 6327 6328 6329 6330 6331 6332 6333
#ifdef CONFIG_XFRM
static void skb_ext_put_sp(struct sec_path *sp)
{
	unsigned int i;

	for (i = 0; i < sp->len; i++)
		xfrm_state_put(sp->xvec[i]);
}
#endif

6334 6335 6336 6337 6338 6339 6340 6341
void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
{
	struct skb_ext *ext = skb->extensions;

	skb->active_extensions &= ~(1 << id);
	if (skb->active_extensions == 0) {
		skb->extensions = NULL;
		__skb_ext_put(ext);
6342 6343 6344 6345 6346 6347 6348 6349
#ifdef CONFIG_XFRM
	} else if (id == SKB_EXT_SEC_PATH &&
		   refcount_read(&ext->refcnt) == 1) {
		struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);

		skb_ext_put_sp(sp);
		sp->len = 0;
#endif
6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364
	}
}
EXPORT_SYMBOL(__skb_ext_del);

void __skb_ext_put(struct skb_ext *ext)
{
	/* If this is last clone, nothing can increment
	 * it after check passes.  Avoids one atomic op.
	 */
	if (refcount_read(&ext->refcnt) == 1)
		goto free_now;

	if (!refcount_dec_and_test(&ext->refcnt))
		return;
free_now:
6365 6366 6367 6368 6369
#ifdef CONFIG_XFRM
	if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
		skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
#endif

6370 6371 6372 6373
	kmem_cache_free(skbuff_ext_cache, ext);
}
EXPORT_SYMBOL(__skb_ext_put);
#endif /* CONFIG_SKB_EXTENSIONS */
新手
引导
客服 返回
顶部