skbuff.c 140.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 *	Routines having to do with the 'struct sk_buff' memory handlers.
 *
4
 *	Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
 *			Florian La Roche <rzsfl@rz.uni-sb.de>
 *
 *	Fixes:
 *		Alan Cox	:	Fixed the worst of the load
 *					balancer bugs.
 *		Dave Platt	:	Interrupt stacking fix.
 *	Richard Kooijman	:	Timestamp fixes.
 *		Alan Cox	:	Changed buffer format.
 *		Alan Cox	:	destructor hook for AF_UNIX etc.
 *		Linus Torvalds	:	Better skb_clone.
 *		Alan Cox	:	Added skb_copy.
 *		Alan Cox	:	Added all the changed routines Linus
 *					only put in the headers
 *		Ray VanTassle	:	Fixed --skb->lock in free
 *		Alan Cox	:	skb_copy copy arp field
 *		Andi Kleen	:	slabified it.
 *		Robert Olsson	:	Removed skb_head_pool
 *
 *	NOTE:
 *		The __skb_ routines should be called with interrupts
 *	disabled, or you better be *real* sure that the operation is atomic
 *	with respect to whatever list is being frobbed (e.g. via lock_sock()
 *	or via disabling bottom half handlers, etc).
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License
 *	as published by the Free Software Foundation; either version
 *	2 of the License, or (at your option) any later version.
 */

/*
 *	The functions in this file will not compile correctly with gcc 2.4.x
 */

J
Joe Perches 已提交
39 40
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/slab.h>
49 50
#include <linux/tcp.h>
#include <linux/udp.h>
M
Marcelo Ricardo Leitner 已提交
51
#include <linux/sctp.h>
L
Linus Torvalds 已提交
52 53 54 55 56 57
#include <linux/netdevice.h>
#ifdef CONFIG_NET_CLS_ACT
#include <net/pkt_sched.h>
#endif
#include <linux/string.h>
#include <linux/skbuff.h>
J
Jens Axboe 已提交
58
#include <linux/splice.h>
L
Linus Torvalds 已提交
59 60 61
#include <linux/cache.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
62
#include <linux/scatterlist.h>
63
#include <linux/errqueue.h>
64
#include <linux/prefetch.h>
65
#include <linux/if_vlan.h>
L
Linus Torvalds 已提交
66 67 68 69 70

#include <net/protocol.h>
#include <net/dst.h>
#include <net/sock.h>
#include <net/checksum.h>
P
Paul Durrant 已提交
71
#include <net/ip6_checksum.h>
L
Linus Torvalds 已提交
72 73
#include <net/xfrm.h>

74
#include <linux/uaccess.h>
75
#include <trace/events/skb.h>
E
Eric Dumazet 已提交
76
#include <linux/highmem.h>
77 78
#include <linux/capability.h>
#include <linux/user_namespace.h>
A
Al Viro 已提交
79

80 81
struct kmem_cache *skbuff_head_cache __ro_after_init;
static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
82 83 84
#ifdef CONFIG_SKB_EXTENSIONS
static struct kmem_cache *skbuff_ext_cache __ro_after_init;
#endif
H
Hans Westgaard Ry 已提交
85 86
int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
EXPORT_SYMBOL(sysctl_max_skb_frags);
L
Linus Torvalds 已提交
87 88

/**
89 90 91 92
 *	skb_panic - private function for out-of-line support
 *	@skb:	buffer
 *	@sz:	size
 *	@addr:	address
93
 *	@msg:	skb_over_panic or skb_under_panic
L
Linus Torvalds 已提交
94
 *
95 96 97 98
 *	Out-of-line support for skb_put() and skb_push().
 *	Called via the wrapper skb_over_panic() or skb_under_panic().
 *	Keep out of line to prevent kernel bloat.
 *	__builtin_return_address is not used because it is not always reliable.
L
Linus Torvalds 已提交
99
 */
100
static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
101
		      const char msg[])
L
Linus Torvalds 已提交
102
{
J
Joe Perches 已提交
103
	pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
104
		 msg, addr, skb->len, sz, skb->head, skb->data,
J
Joe Perches 已提交
105 106
		 (unsigned long)skb->tail, (unsigned long)skb->end,
		 skb->dev ? skb->dev->name : "<NULL>");
L
Linus Torvalds 已提交
107 108 109
	BUG();
}

110
static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
L
Linus Torvalds 已提交
111
{
112
	skb_panic(skb, sz, addr, __func__);
L
Linus Torvalds 已提交
113 114
}

115 116 117 118
static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
{
	skb_panic(skb, sz, addr, __func__);
}
119 120 121 122 123 124 125 126 127 128

/*
 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
 * the caller if emergency pfmemalloc reserves are being used. If it is and
 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
 * may be used. Otherwise, the packet data may be discarded until enough
 * memory is free
 */
#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
	 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
129 130 131

static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
			       unsigned long ip, bool *pfmemalloc)
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
{
	void *obj;
	bool ret_pfmemalloc = false;

	/*
	 * Try a regular allocation, when that fails and we're not entitled
	 * to the reserves, fail.
	 */
	obj = kmalloc_node_track_caller(size,
					flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
					node);
	if (obj || !(gfp_pfmemalloc_allowed(flags)))
		goto out;

	/* Try again but now we are using pfmemalloc reserves */
	ret_pfmemalloc = true;
	obj = kmalloc_node_track_caller(size, flags, node);

out:
	if (pfmemalloc)
		*pfmemalloc = ret_pfmemalloc;

	return obj;
}

L
Linus Torvalds 已提交
157 158 159 160 161 162 163
/* 	Allocate a new skbuff. We do this ourselves so we can fill in a few
 *	'private' fields and also do memory statistics to find all the
 *	[BEEP] leaks.
 *
 */

/**
164
 *	__alloc_skb	-	allocate a network buffer
L
Linus Torvalds 已提交
165 166
 *	@size: size to allocate
 *	@gfp_mask: allocation mask
167 168 169 170
 *	@flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
 *		instead of head cache and allocate a cloned (child) skb.
 *		If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
 *		allocations in case the data is required for writeback
171
 *	@node: numa node to allocate memory on
L
Linus Torvalds 已提交
172 173
 *
 *	Allocate a new &sk_buff. The returned buffer has no headroom and a
174 175
 *	tail room of at least size bytes. The object has a reference count
 *	of one. The return is the buffer. On a failure the return is %NULL.
L
Linus Torvalds 已提交
176 177 178 179
 *
 *	Buffers may only be allocated from interrupts using a @gfp_mask of
 *	%GFP_ATOMIC.
 */
A
Al Viro 已提交
180
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
181
			    int flags, int node)
L
Linus Torvalds 已提交
182
{
183
	struct kmem_cache *cache;
B
Benjamin LaHaise 已提交
184
	struct skb_shared_info *shinfo;
L
Linus Torvalds 已提交
185 186
	struct sk_buff *skb;
	u8 *data;
187
	bool pfmemalloc;
L
Linus Torvalds 已提交
188

189 190 191 192 193
	cache = (flags & SKB_ALLOC_FCLONE)
		? skbuff_fclone_cache : skbuff_head_cache;

	if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
		gfp_mask |= __GFP_MEMALLOC;
194

L
Linus Torvalds 已提交
195
	/* Get the HEAD */
196
	skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
L
Linus Torvalds 已提交
197 198
	if (!skb)
		goto out;
E
Eric Dumazet 已提交
199
	prefetchw(skb);
L
Linus Torvalds 已提交
200

E
Eric Dumazet 已提交
201 202 203 204 205
	/* We do our best to align skb_shared_info on a separate cache
	 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
	 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
	 * Both skb->head and skb_shared_info are cache line aligned.
	 */
206
	size = SKB_DATA_ALIGN(size);
E
Eric Dumazet 已提交
207
	size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
208
	data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
L
Linus Torvalds 已提交
209 210
	if (!data)
		goto nodata;
E
Eric Dumazet 已提交
211 212 213 214 215
	/* kmalloc(size) might give us more room than requested.
	 * Put skb_shared_info exactly at the end of allocated zone,
	 * to allow max possible filling before reallocation.
	 */
	size = SKB_WITH_OVERHEAD(ksize(data));
E
Eric Dumazet 已提交
216
	prefetchw(data + size);
L
Linus Torvalds 已提交
217

218
	/*
219 220 221
	 * Only clear those fields we need to clear, not those that we will
	 * actually initialise below. Hence, don't put any more fields after
	 * the tail pointer in struct sk_buff!
222 223
	 */
	memset(skb, 0, offsetof(struct sk_buff, tail));
E
Eric Dumazet 已提交
224 225
	/* Account for allocated memory : skb + skb->head */
	skb->truesize = SKB_TRUESIZE(size);
226
	skb->pfmemalloc = pfmemalloc;
227
	refcount_set(&skb->users, 1);
L
Linus Torvalds 已提交
228 229
	skb->head = data;
	skb->data = data;
230
	skb_reset_tail_pointer(skb);
231
	skb->end = skb->tail + size;
C
Cong Wang 已提交
232 233
	skb->mac_header = (typeof(skb->mac_header))~0U;
	skb->transport_header = (typeof(skb->transport_header))~0U;
234

B
Benjamin LaHaise 已提交
235 236
	/* make sure we initialize shinfo sequentially */
	shinfo = skb_shinfo(skb);
E
Eric Dumazet 已提交
237
	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
B
Benjamin LaHaise 已提交
238 239
	atomic_set(&shinfo->dataref, 1);

240
	if (flags & SKB_ALLOC_FCLONE) {
241
		struct sk_buff_fclones *fclones;
L
Linus Torvalds 已提交
242

243 244
		fclones = container_of(skb, struct sk_buff_fclones, skb1);

245
		skb->fclone = SKB_FCLONE_ORIG;
246
		refcount_set(&fclones->fclone_ref, 1);
247

248
		fclones->skb2.fclone = SKB_FCLONE_CLONE;
249
	}
L
Linus Torvalds 已提交
250 251 252
out:
	return skb;
nodata:
253
	kmem_cache_free(cache, skb);
L
Linus Torvalds 已提交
254 255 256
	skb = NULL;
	goto out;
}
257
EXPORT_SYMBOL(__alloc_skb);
L
Linus Torvalds 已提交
258

E
Eric Dumazet 已提交
259
/**
E
Eric Dumazet 已提交
260
 * __build_skb - build a network buffer
E
Eric Dumazet 已提交
261
 * @data: data buffer provided by caller
E
Eric Dumazet 已提交
262
 * @frag_size: size of data, or 0 if head was kmalloced
E
Eric Dumazet 已提交
263 264
 *
 * Allocate a new &sk_buff. Caller provides space holding head and
265
 * skb_shared_info. @data must have been allocated by kmalloc() only if
E
Eric Dumazet 已提交
266 267
 * @frag_size is 0, otherwise data should come from the page allocator
 *  or vmalloc()
E
Eric Dumazet 已提交
268 269 270 271 272 273 274 275 276 277
 * The return is the new skb buffer.
 * On a failure the return is %NULL, and @data is not freed.
 * Notes :
 *  Before IO, driver allocates only data buffer where NIC put incoming frame
 *  Driver should add room at head (NET_SKB_PAD) and
 *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
 *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
 *  before giving packet to stack.
 *  RX rings only contains data buffers, not full skbs.
 */
E
Eric Dumazet 已提交
278
struct sk_buff *__build_skb(void *data, unsigned int frag_size)
E
Eric Dumazet 已提交
279 280 281
{
	struct skb_shared_info *shinfo;
	struct sk_buff *skb;
282
	unsigned int size = frag_size ? : ksize(data);
E
Eric Dumazet 已提交
283 284 285 286 287

	skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
	if (!skb)
		return NULL;

288
	size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
E
Eric Dumazet 已提交
289 290 291

	memset(skb, 0, offsetof(struct sk_buff, tail));
	skb->truesize = SKB_TRUESIZE(size);
292
	refcount_set(&skb->users, 1);
E
Eric Dumazet 已提交
293 294 295 296
	skb->head = data;
	skb->data = data;
	skb_reset_tail_pointer(skb);
	skb->end = skb->tail + size;
C
Cong Wang 已提交
297 298
	skb->mac_header = (typeof(skb->mac_header))~0U;
	skb->transport_header = (typeof(skb->transport_header))~0U;
E
Eric Dumazet 已提交
299 300 301 302 303 304 305 306

	/* make sure we initialize shinfo sequentially */
	shinfo = skb_shinfo(skb);
	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
	atomic_set(&shinfo->dataref, 1);

	return skb;
}
E
Eric Dumazet 已提交
307 308 309 310 311 312 313 314 315 316 317 318

/* build_skb() is wrapper over __build_skb(), that specifically
 * takes care of skb->head and skb->pfmemalloc
 * This means that if @frag_size is not zero, then @data must be backed
 * by a page fragment, not kmalloc() or vmalloc()
 */
struct sk_buff *build_skb(void *data, unsigned int frag_size)
{
	struct sk_buff *skb = __build_skb(data, frag_size);

	if (skb && frag_size) {
		skb->head_frag = 1;
319
		if (page_is_pfmemalloc(virt_to_head_page(data)))
E
Eric Dumazet 已提交
320 321 322 323
			skb->pfmemalloc = 1;
	}
	return skb;
}
E
Eric Dumazet 已提交
324 325
EXPORT_SYMBOL(build_skb);

326 327 328 329
#define NAPI_SKB_CACHE_SIZE	64

struct napi_alloc_cache {
	struct page_frag_cache page;
330
	unsigned int skb_count;
331 332 333
	void *skb_cache[NAPI_SKB_CACHE_SIZE];
};

334
static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
335
static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
336 337 338

static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
{
339
	struct page_frag_cache *nc;
340 341 342 343
	unsigned long flags;
	void *data;

	local_irq_save(flags);
344
	nc = this_cpu_ptr(&netdev_alloc_cache);
345
	data = page_frag_alloc(nc, fragsz, gfp_mask);
346 347 348
	local_irq_restore(flags);
	return data;
}
349 350 351 352 353 354 355 356 357 358

/**
 * netdev_alloc_frag - allocate a page fragment
 * @fragsz: fragment size
 *
 * Allocates a frag from a page for receive buffer.
 * Uses GFP_ATOMIC allocations.
 */
void *netdev_alloc_frag(unsigned int fragsz)
{
M
Mel Gorman 已提交
359
	return __netdev_alloc_frag(fragsz, GFP_ATOMIC);
360
}
361 362
EXPORT_SYMBOL(netdev_alloc_frag);

363 364
static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
{
365
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
366

367
	return page_frag_alloc(&nc->page, fragsz, gfp_mask);
368 369 370 371
}

void *napi_alloc_frag(unsigned int fragsz)
{
M
Mel Gorman 已提交
372
	return __napi_alloc_frag(fragsz, GFP_ATOMIC);
373 374 375
}
EXPORT_SYMBOL(napi_alloc_frag);

376 377 378
/**
 *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device
 *	@dev: network device to receive on
379
 *	@len: length to allocate
380 381 382 383 384 385 386 387 388
 *	@gfp_mask: get_free_pages mask, passed to alloc_skb
 *
 *	Allocate a new &sk_buff and assign it a usage count of one. The
 *	buffer has NET_SKB_PAD headroom built in. Users should allocate
 *	the headroom they think they need without accounting for the
 *	built in space. The built in space is used for optimisations.
 *
 *	%NULL is returned if there is no free memory.
 */
389 390
struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
				   gfp_t gfp_mask)
391
{
392
	struct page_frag_cache *nc;
393
	unsigned long flags;
394
	struct sk_buff *skb;
395 396 397 398
	bool pfmemalloc;
	void *data;

	len += NET_SKB_PAD;
399

400
	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
401
	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
402 403 404 405 406
		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
		if (!skb)
			goto skb_fail;
		goto skb_success;
	}
407

408 409 410 411 412 413 414 415 416
	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	len = SKB_DATA_ALIGN(len);

	if (sk_memalloc_socks())
		gfp_mask |= __GFP_MEMALLOC;

	local_irq_save(flags);

	nc = this_cpu_ptr(&netdev_alloc_cache);
417
	data = page_frag_alloc(nc, len, gfp_mask);
418 419 420 421 422 423 424 425 426
	pfmemalloc = nc->pfmemalloc;

	local_irq_restore(flags);

	if (unlikely(!data))
		return NULL;

	skb = __build_skb(data, len);
	if (unlikely(!skb)) {
427
		skb_free_frag(data);
428
		return NULL;
429
	}
430

431 432 433 434 435
	/* use OR instead of assignment to avoid clearing of bits in mask */
	if (pfmemalloc)
		skb->pfmemalloc = 1;
	skb->head_frag = 1;

436
skb_success:
437 438 439
	skb_reserve(skb, NET_SKB_PAD);
	skb->dev = dev;

440
skb_fail:
441 442
	return skb;
}
443
EXPORT_SYMBOL(__netdev_alloc_skb);
L
Linus Torvalds 已提交
444

445 446 447
/**
 *	__napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
 *	@napi: napi instance this buffer was allocated for
448
 *	@len: length to allocate
449 450 451 452 453 454 455 456 457
 *	@gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
 *
 *	Allocate a new sk_buff for use in NAPI receive.  This buffer will
 *	attempt to allocate the head from a special reserved region used
 *	only for NAPI Rx allocation.  By doing this we can save several
 *	CPU cycles by avoiding having to disable and re-enable IRQs.
 *
 *	%NULL is returned if there is no free memory.
 */
458 459
struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
				 gfp_t gfp_mask)
460
{
461
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
462
	struct sk_buff *skb;
463 464 465
	void *data;

	len += NET_SKB_PAD + NET_IP_ALIGN;
466

467
	if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
468
	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
469 470 471 472 473
		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
		if (!skb)
			goto skb_fail;
		goto skb_success;
	}
474 475 476 477 478 479

	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	len = SKB_DATA_ALIGN(len);

	if (sk_memalloc_socks())
		gfp_mask |= __GFP_MEMALLOC;
480

481
	data = page_frag_alloc(&nc->page, len, gfp_mask);
482 483 484 485 486
	if (unlikely(!data))
		return NULL;

	skb = __build_skb(data, len);
	if (unlikely(!skb)) {
487
		skb_free_frag(data);
488
		return NULL;
489 490
	}

491
	/* use OR instead of assignment to avoid clearing of bits in mask */
492
	if (nc->page.pfmemalloc)
493 494 495
		skb->pfmemalloc = 1;
	skb->head_frag = 1;

496
skb_success:
497 498 499
	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
	skb->dev = napi->dev;

500
skb_fail:
501 502 503 504
	return skb;
}
EXPORT_SYMBOL(__napi_alloc_skb);

P
Peter Zijlstra 已提交
505
void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
506
		     int size, unsigned int truesize)
P
Peter Zijlstra 已提交
507 508 509 510
{
	skb_fill_page_desc(skb, i, page, off, size);
	skb->len += size;
	skb->data_len += size;
511
	skb->truesize += truesize;
P
Peter Zijlstra 已提交
512 513 514
}
EXPORT_SYMBOL(skb_add_rx_frag);

J
Jason Wang 已提交
515 516 517 518 519 520 521 522 523 524 525 526
void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
			  unsigned int truesize)
{
	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

	skb_frag_size_add(frag, size);
	skb->len += size;
	skb->data_len += size;
	skb->truesize += truesize;
}
EXPORT_SYMBOL(skb_coalesce_rx_frag);

527
static void skb_drop_list(struct sk_buff **listp)
L
Linus Torvalds 已提交
528
{
E
Eric Dumazet 已提交
529
	kfree_skb_list(*listp);
530
	*listp = NULL;
L
Linus Torvalds 已提交
531 532
}

533 534 535 536 537
static inline void skb_drop_fraglist(struct sk_buff *skb)
{
	skb_drop_list(&skb_shinfo(skb)->frag_list);
}

L
Linus Torvalds 已提交
538 539 540 541
static void skb_clone_fraglist(struct sk_buff *skb)
{
	struct sk_buff *list;

542
	skb_walk_frags(skb, list)
L
Linus Torvalds 已提交
543 544 545
		skb_get(list);
}

546 547
static void skb_free_head(struct sk_buff *skb)
{
548 549
	unsigned char *head = skb->head;

550
	if (skb->head_frag)
551
		skb_free_frag(head);
552
	else
553
		kfree(head);
554 555
}

556
static void skb_release_data(struct sk_buff *skb)
L
Linus Torvalds 已提交
557
{
E
Eric Dumazet 已提交
558 559
	struct skb_shared_info *shinfo = skb_shinfo(skb);
	int i;
L
Linus Torvalds 已提交
560

E
Eric Dumazet 已提交
561 562 563 564
	if (skb->cloned &&
	    atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
			      &shinfo->dataref))
		return;
565

E
Eric Dumazet 已提交
566 567
	for (i = 0; i < shinfo->nr_frags; i++)
		__skb_frag_unref(&shinfo->frags[i]);
568

E
Eric Dumazet 已提交
569 570 571
	if (shinfo->frag_list)
		kfree_skb_list(shinfo->frag_list);

W
Willem de Bruijn 已提交
572
	skb_zcopy_clear(skb, true);
E
Eric Dumazet 已提交
573
	skb_free_head(skb);
L
Linus Torvalds 已提交
574 575 576 577 578
}

/*
 *	Free an skbuff by memory without cleaning the state.
 */
579
static void kfree_skbmem(struct sk_buff *skb)
L
Linus Torvalds 已提交
580
{
581
	struct sk_buff_fclones *fclones;
582 583 584 585

	switch (skb->fclone) {
	case SKB_FCLONE_UNAVAILABLE:
		kmem_cache_free(skbuff_head_cache, skb);
586
		return;
587 588

	case SKB_FCLONE_ORIG:
589
		fclones = container_of(skb, struct sk_buff_fclones, skb1);
590

591 592 593
		/* We usually free the clone (TX completion) before original skb
		 * This test would have no chance to be true for the clone,
		 * while here, branch prediction will be good.
594
		 */
595
		if (refcount_read(&fclones->fclone_ref) == 1)
596 597
			goto fastpath;
		break;
598

599 600
	default: /* SKB_FCLONE_CLONE */
		fclones = container_of(skb, struct sk_buff_fclones, skb2);
601
		break;
602
	}
603
	if (!refcount_dec_and_test(&fclones->fclone_ref))
604 605 606
		return;
fastpath:
	kmem_cache_free(skbuff_fclone_cache, fclones);
L
Linus Torvalds 已提交
607 608
}

P
Paolo Abeni 已提交
609
void skb_release_head_state(struct sk_buff *skb)
L
Linus Torvalds 已提交
610
{
E
Eric Dumazet 已提交
611
	skb_dst_drop(skb);
P
Paolo Abeni 已提交
612
	secpath_reset(skb);
613 614
	if (skb->destructor) {
		WARN_ON(in_irq());
L
Linus Torvalds 已提交
615 616
		skb->destructor(skb);
	}
I
Igor Maravić 已提交
617
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
618
	nf_conntrack_put(skb_nfct(skb));
L
Linus Torvalds 已提交
619
#endif
620
	skb_ext_put(skb);
621 622 623 624 625 626
}

/* Free everything but the sk_buff shell. */
static void skb_release_all(struct sk_buff *skb)
{
	skb_release_head_state(skb);
627 628
	if (likely(skb->head))
		skb_release_data(skb);
629 630 631 632 633 634 635 636 637 638
}

/**
 *	__kfree_skb - private function
 *	@skb: buffer
 *
 *	Free an sk_buff. Release anything attached to the buffer.
 *	Clean the state. This is an internal helper function. Users should
 *	always call kfree_skb
 */
L
Linus Torvalds 已提交
639

640 641 642
void __kfree_skb(struct sk_buff *skb)
{
	skb_release_all(skb);
L
Linus Torvalds 已提交
643 644
	kfree_skbmem(skb);
}
645
EXPORT_SYMBOL(__kfree_skb);
L
Linus Torvalds 已提交
646

647 648 649 650 651 652 653 654 655
/**
 *	kfree_skb - free an sk_buff
 *	@skb: buffer to free
 *
 *	Drop a reference to the buffer and free it if the usage count has
 *	hit zero.
 */
void kfree_skb(struct sk_buff *skb)
{
656
	if (!skb_unref(skb))
657
		return;
658

659
	trace_kfree_skb(skb, __builtin_return_address(0));
660 661
	__kfree_skb(skb);
}
662
EXPORT_SYMBOL(kfree_skb);
663

E
Eric Dumazet 已提交
664 665 666 667 668 669 670 671 672 673 674
void kfree_skb_list(struct sk_buff *segs)
{
	while (segs) {
		struct sk_buff *next = segs->next;

		kfree_skb(segs);
		segs = next;
	}
}
EXPORT_SYMBOL(kfree_skb_list);

675 676 677 678 679 680 681 682 683
/**
 *	skb_tx_error - report an sk_buff xmit error
 *	@skb: buffer that triggered an error
 *
 *	Report xmit error if a device callback is tracking this skb.
 *	skb must be freed afterwards.
 */
void skb_tx_error(struct sk_buff *skb)
{
W
Willem de Bruijn 已提交
684
	skb_zcopy_clear(skb, true);
685 686 687
}
EXPORT_SYMBOL(skb_tx_error);

688 689 690 691 692 693 694 695 696 697
/**
 *	consume_skb - free an skbuff
 *	@skb: buffer to free
 *
 *	Drop a ref to the buffer and free it if the usage count has hit zero
 *	Functions identically to kfree_skb, but kfree_skb assumes that the frame
 *	is being dropped after a failure and notes that
 */
void consume_skb(struct sk_buff *skb)
{
698
	if (!skb_unref(skb))
699
		return;
700

701
	trace_consume_skb(skb);
702 703 704 705
	__kfree_skb(skb);
}
EXPORT_SYMBOL(consume_skb);

P
Paolo Abeni 已提交
706 707 708 709
/**
 *	consume_stateless_skb - free an skbuff, assuming it is stateless
 *	@skb: buffer to free
 *
710 711
 *	Alike consume_skb(), but this variant assumes that this is the last
 *	skb reference and all the head states have been already dropped
P
Paolo Abeni 已提交
712
 */
713
void __consume_stateless_skb(struct sk_buff *skb)
P
Paolo Abeni 已提交
714 715
{
	trace_consume_skb(skb);
716
	skb_release_data(skb);
P
Paolo Abeni 已提交
717 718 719
	kfree_skbmem(skb);
}

720 721 722 723 724 725 726 727 728 729 730 731
void __kfree_skb_flush(void)
{
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);

	/* flush skb_cache if containing objects */
	if (nc->skb_count) {
		kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
				     nc->skb_cache);
		nc->skb_count = 0;
	}
}

732
static inline void _kfree_skb_defer(struct sk_buff *skb)
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
{
	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);

	/* drop skb->head and call any destructors for packet */
	skb_release_all(skb);

	/* record skb to CPU local list */
	nc->skb_cache[nc->skb_count++] = skb;

#ifdef CONFIG_SLUB
	/* SLUB writes into objects when freeing */
	prefetchw(skb);
#endif

	/* flush skb_cache if it is filled */
	if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
		kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
				     nc->skb_cache);
		nc->skb_count = 0;
	}
}
754 755 756 757
void __kfree_skb_defer(struct sk_buff *skb)
{
	_kfree_skb_defer(skb);
}
758 759 760 761 762 763

void napi_consume_skb(struct sk_buff *skb, int budget)
{
	if (unlikely(!skb))
		return;

764
	/* Zero budget indicate non-NAPI context called us, like netpoll */
765
	if (unlikely(!budget)) {
766
		dev_consume_skb_any(skb);
767 768 769
		return;
	}

770
	if (!skb_unref(skb))
771
		return;
772

773 774 775 776
	/* if reaching here SKB is ready to free */
	trace_consume_skb(skb);

	/* if SKB is a clone, don't handle this case */
777
	if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
778 779 780 781
		__kfree_skb(skb);
		return;
	}

782
	_kfree_skb_defer(skb);
783 784 785
}
EXPORT_SYMBOL(napi_consume_skb);

786 787 788 789 790 791 792
/* Make sure a field is enclosed inside headers_start/headers_end section */
#define CHECK_SKB_FIELD(field) \
	BUILD_BUG_ON(offsetof(struct sk_buff, field) <		\
		     offsetof(struct sk_buff, headers_start));	\
	BUILD_BUG_ON(offsetof(struct sk_buff, field) >		\
		     offsetof(struct sk_buff, headers_end));	\

793 794 795
static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
{
	new->tstamp		= old->tstamp;
796
	/* We do not copy old->sk */
797
	new->dev		= old->dev;
798
	memcpy(new->cb, old->cb, sizeof(old->cb));
E
Eric Dumazet 已提交
799
	skb_dst_copy(new, old);
800
	__skb_ext_copy(new, old);
801
#ifdef CONFIG_XFRM
802 803
	new->sp			= secpath_get(old->sp);
#endif
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833
	__nf_copy(new, old, false);

	/* Note : this field could be in headers_start/headers_end section
	 * It is not yet because we do not want to have a 16 bit hole
	 */
	new->queue_mapping = old->queue_mapping;

	memcpy(&new->headers_start, &old->headers_start,
	       offsetof(struct sk_buff, headers_end) -
	       offsetof(struct sk_buff, headers_start));
	CHECK_SKB_FIELD(protocol);
	CHECK_SKB_FIELD(csum);
	CHECK_SKB_FIELD(hash);
	CHECK_SKB_FIELD(priority);
	CHECK_SKB_FIELD(skb_iif);
	CHECK_SKB_FIELD(vlan_proto);
	CHECK_SKB_FIELD(vlan_tci);
	CHECK_SKB_FIELD(transport_header);
	CHECK_SKB_FIELD(network_header);
	CHECK_SKB_FIELD(mac_header);
	CHECK_SKB_FIELD(inner_protocol);
	CHECK_SKB_FIELD(inner_transport_header);
	CHECK_SKB_FIELD(inner_network_header);
	CHECK_SKB_FIELD(inner_mac_header);
	CHECK_SKB_FIELD(mark);
#ifdef CONFIG_NETWORK_SECMARK
	CHECK_SKB_FIELD(secmark);
#endif
#ifdef CONFIG_NET_RX_BUSY_POLL
	CHECK_SKB_FIELD(napi_id);
834
#endif
E
Eric Dumazet 已提交
835 836 837
#ifdef CONFIG_XPS
	CHECK_SKB_FIELD(sender_cpu);
#endif
838
#ifdef CONFIG_NET_SCHED
839
	CHECK_SKB_FIELD(tc_index);
840
#endif
E
Eliezer Tamir 已提交
841

842 843
}

844 845 846 847
/*
 * You should not add any new code to this function.  Add it to
 * __copy_skb_header above instead.
 */
H
Herbert Xu 已提交
848
static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
L
Linus Torvalds 已提交
849 850 851 852 853
{
#define C(x) n->x = skb->x

	n->next = n->prev = NULL;
	n->sk = NULL;
854 855
	__copy_skb_header(n, skb);

L
Linus Torvalds 已提交
856 857
	C(len);
	C(data_len);
858
	C(mac_len);
859
	n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
860
	n->cloned = 1;
L
Linus Torvalds 已提交
861
	n->nohdr = 0;
862
	n->peeked = 0;
863
	C(pfmemalloc);
L
Linus Torvalds 已提交
864 865 866
	n->destructor = NULL;
	C(tail);
	C(end);
867
	C(head);
868
	C(head_frag);
869 870
	C(data);
	C(truesize);
871
	refcount_set(&n->users, 1);
L
Linus Torvalds 已提交
872 873 874 875 876

	atomic_inc(&(skb_shinfo(skb)->dataref));
	skb->cloned = 1;

	return n;
H
Herbert Xu 已提交
877 878 879 880 881 882 883 884 885 886 887 888 889 890 891
#undef C
}

/**
 *	skb_morph	-	morph one skb into another
 *	@dst: the skb to receive the contents
 *	@src: the skb to supply the contents
 *
 *	This is identical to skb_clone except that the target skb is
 *	supplied by the user.
 *
 *	The target skb is returned upon exit.
 */
struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
{
892
	skb_release_all(dst);
H
Herbert Xu 已提交
893 894 895 896
	return __skb_clone(dst, src);
}
EXPORT_SYMBOL_GPL(skb_morph);

897
int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925
{
	unsigned long max_pg, num_pg, new_pg, old_pg;
	struct user_struct *user;

	if (capable(CAP_IPC_LOCK) || !size)
		return 0;

	num_pg = (size >> PAGE_SHIFT) + 2;	/* worst case */
	max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	user = mmp->user ? : current_user();

	do {
		old_pg = atomic_long_read(&user->locked_vm);
		new_pg = old_pg + num_pg;
		if (new_pg > max_pg)
			return -ENOBUFS;
	} while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
		 old_pg);

	if (!mmp->user) {
		mmp->user = get_uid(user);
		mmp->num_pg = num_pg;
	} else {
		mmp->num_pg += num_pg;
	}

	return 0;
}
926
EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
927

928
void mm_unaccount_pinned_pages(struct mmpin *mmp)
929 930 931 932 933 934
{
	if (mmp->user) {
		atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
		free_uid(mmp->user);
	}
}
935
EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
936

W
Willem de Bruijn 已提交
937 938 939 940 941 942 943 944 945 946 947 948 949
struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
{
	struct ubuf_info *uarg;
	struct sk_buff *skb;

	WARN_ON_ONCE(!in_task());

	skb = sock_omalloc(sk, 0, GFP_KERNEL);
	if (!skb)
		return NULL;

	BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
	uarg = (void *)skb->cb;
950 951 952 953 954 955
	uarg->mmp.user = NULL;

	if (mm_account_pinned_pages(&uarg->mmp, size)) {
		kfree_skb(skb);
		return NULL;
	}
W
Willem de Bruijn 已提交
956 957

	uarg->callback = sock_zerocopy_callback;
958 959 960
	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
	uarg->len = 1;
	uarg->bytelen = size;
W
Willem de Bruijn 已提交
961
	uarg->zerocopy = 1;
962
	refcount_set(&uarg->refcnt, 1);
W
Willem de Bruijn 已提交
963 964 965 966 967 968 969 970 971 972 973
	sock_hold(sk);

	return uarg;
}
EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);

static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
{
	return container_of((void *)uarg, struct sk_buff, cb);
}

974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
					struct ubuf_info *uarg)
{
	if (uarg) {
		const u32 byte_limit = 1 << 19;		/* limit to a few TSO */
		u32 bytelen, next;

		/* realloc only when socket is locked (TCP, UDP cork),
		 * so uarg->len and sk_zckey access is serialized
		 */
		if (!sock_owned_by_user(sk)) {
			WARN_ON_ONCE(1);
			return NULL;
		}

		bytelen = uarg->bytelen + size;
		if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
			/* TCP can create new skb to attach new uarg */
			if (sk->sk_type == SOCK_STREAM)
				goto new_alloc;
			return NULL;
		}

		next = (u32)atomic_read(&sk->sk_zckey);
		if ((u32)(uarg->id + uarg->len) == next) {
999 1000
			if (mm_account_pinned_pages(&uarg->mmp, size))
				return NULL;
1001 1002 1003
			uarg->len++;
			uarg->bytelen = bytelen;
			atomic_set(&sk->sk_zckey, ++next);
1004
			sock_zerocopy_get(uarg);
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
			return uarg;
		}
	}

new_alloc:
	return sock_zerocopy_alloc(sk, size);
}
EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);

static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
{
	struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
	u32 old_lo, old_hi;
	u64 sum_len;

	old_lo = serr->ee.ee_info;
	old_hi = serr->ee.ee_data;
	sum_len = old_hi - old_lo + 1ULL + len;

	if (sum_len >= (1ULL << 32))
		return false;

	if (lo != old_hi + 1)
		return false;

	serr->ee.ee_data += len;
	return true;
}

W
Willem de Bruijn 已提交
1034 1035
void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
{
1036
	struct sk_buff *tail, *skb = skb_from_uarg(uarg);
W
Willem de Bruijn 已提交
1037 1038
	struct sock_exterr_skb *serr;
	struct sock *sk = skb->sk;
1039 1040 1041 1042
	struct sk_buff_head *q;
	unsigned long flags;
	u32 lo, hi;
	u16 len;
W
Willem de Bruijn 已提交
1043

1044 1045
	mm_unaccount_pinned_pages(&uarg->mmp);

1046 1047 1048 1049
	/* if !len, there was only 1 call, and it was aborted
	 * so do not queue a completion notification
	 */
	if (!uarg->len || sock_flag(sk, SOCK_DEAD))
W
Willem de Bruijn 已提交
1050 1051
		goto release;

1052 1053 1054 1055
	len = uarg->len;
	lo = uarg->id;
	hi = uarg->id + len - 1;

W
Willem de Bruijn 已提交
1056 1057 1058 1059
	serr = SKB_EXT_ERR(skb);
	memset(serr, 0, sizeof(*serr));
	serr->ee.ee_errno = 0;
	serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1060 1061
	serr->ee.ee_data = hi;
	serr->ee.ee_info = lo;
W
Willem de Bruijn 已提交
1062 1063 1064
	if (!success)
		serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;

1065 1066 1067 1068 1069 1070 1071 1072 1073
	q = &sk->sk_error_queue;
	spin_lock_irqsave(&q->lock, flags);
	tail = skb_peek_tail(q);
	if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
	    !skb_zerocopy_notify_extend(tail, lo, len)) {
		__skb_queue_tail(q, skb);
		skb = NULL;
	}
	spin_unlock_irqrestore(&q->lock, flags);
W
Willem de Bruijn 已提交
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084

	sk->sk_error_report(sk);

release:
	consume_skb(skb);
	sock_put(sk);
}
EXPORT_SYMBOL_GPL(sock_zerocopy_callback);

void sock_zerocopy_put(struct ubuf_info *uarg)
{
1085
	if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
W
Willem de Bruijn 已提交
1086 1087 1088 1089 1090 1091 1092 1093
		if (uarg->callback)
			uarg->callback(uarg, uarg->zerocopy);
		else
			consume_skb(skb_from_uarg(uarg));
	}
}
EXPORT_SYMBOL_GPL(sock_zerocopy_put);

1094
void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
W
Willem de Bruijn 已提交
1095 1096 1097 1098 1099
{
	if (uarg) {
		struct sock *sk = skb_from_uarg(uarg)->sk;

		atomic_dec(&sk->sk_zckey);
1100
		uarg->len--;
W
Willem de Bruijn 已提交
1101

1102 1103
		if (have_uref)
			sock_zerocopy_put(uarg);
W
Willem de Bruijn 已提交
1104 1105 1106 1107 1108 1109 1110
	}
}
EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);

extern int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,
				   struct iov_iter *from, size_t length);

W
Willem de Bruijn 已提交
1111 1112 1113 1114 1115 1116
int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
{
	return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
}
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);

W
Willem de Bruijn 已提交
1117 1118 1119 1120
int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
			     struct msghdr *msg, int len,
			     struct ubuf_info *uarg)
{
1121
	struct ubuf_info *orig_uarg = skb_zcopy(skb);
W
Willem de Bruijn 已提交
1122 1123 1124
	struct iov_iter orig_iter = msg->msg_iter;
	int err, orig_len = skb->len;

1125 1126 1127 1128 1129 1130
	/* An skb can only point to one uarg. This edge case happens when
	 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
	 */
	if (orig_uarg && uarg != orig_uarg)
		return -EEXIST;

W
Willem de Bruijn 已提交
1131 1132
	err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
	if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1133 1134
		struct sock *save_sk = skb->sk;

W
Willem de Bruijn 已提交
1135 1136
		/* Streams do not free skb on error. Reset to prev state. */
		msg->msg_iter = orig_iter;
1137
		skb->sk = sk;
W
Willem de Bruijn 已提交
1138
		___pskb_trim(skb, orig_len);
1139
		skb->sk = save_sk;
W
Willem de Bruijn 已提交
1140 1141 1142
		return err;
	}

1143
	skb_zcopy_set(skb, uarg, NULL);
W
Willem de Bruijn 已提交
1144 1145 1146 1147
	return skb->len - orig_len;
}
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);

W
Willem de Bruijn 已提交
1148
static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
W
Willem de Bruijn 已提交
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
			      gfp_t gfp_mask)
{
	if (skb_zcopy(orig)) {
		if (skb_zcopy(nskb)) {
			/* !gfp_mask callers are verified to !skb_zcopy(nskb) */
			if (!gfp_mask) {
				WARN_ON_ONCE(1);
				return -ENOMEM;
			}
			if (skb_uarg(nskb) == skb_uarg(orig))
				return 0;
			if (skb_copy_ubufs(nskb, GFP_ATOMIC))
				return -EIO;
		}
1163
		skb_zcopy_set(nskb, skb_uarg(orig), NULL);
W
Willem de Bruijn 已提交
1164 1165 1166 1167
	}
	return 0;
}

1168 1169
/**
 *	skb_copy_ubufs	-	copy userspace skb frags buffers to kernel
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
 *	@skb: the skb to modify
 *	@gfp_mask: allocation priority
 *
 *	This must be called on SKBTX_DEV_ZEROCOPY skb.
 *	It will copy all frags into kernel and drop the reference
 *	to userspace pages.
 *
 *	If this function is called from an interrupt gfp_mask() must be
 *	%GFP_ATOMIC.
 *
 *	Returns 0 on success or a negative error code on failure
 *	to allocate kernel memory to copy to.
 */
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1184 1185 1186
{
	int num_frags = skb_shinfo(skb)->nr_frags;
	struct page *page, *head = NULL;
1187 1188
	int i, new_frags;
	u32 d_off;
1189

1190 1191
	if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
		return -EINVAL;
1192

1193 1194 1195
	if (!num_frags)
		goto release;

1196 1197
	new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
	for (i = 0; i < new_frags; i++) {
1198
		page = alloc_page(gfp_mask);
1199 1200
		if (!page) {
			while (head) {
1201
				struct page *next = (struct page *)page_private(head);
1202 1203 1204 1205 1206
				put_page(head);
				head = next;
			}
			return -ENOMEM;
		}
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217
		set_page_private(page, (unsigned long)head);
		head = page;
	}

	page = head;
	d_off = 0;
	for (i = 0; i < num_frags; i++) {
		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
		u32 p_off, p_len, copied;
		struct page *p;
		u8 *vaddr;
1218 1219 1220

		skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
				      p, p_off, p_len, copied) {
1221
			u32 copy, done = 0;
1222
			vaddr = kmap_atomic(p);
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234

			while (done < p_len) {
				if (d_off == PAGE_SIZE) {
					d_off = 0;
					page = (struct page *)page_private(page);
				}
				copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
				memcpy(page_address(page) + d_off,
				       vaddr + p_off + done, copy);
				done += copy;
				d_off += copy;
			}
1235 1236
			kunmap_atomic(vaddr);
		}
1237 1238 1239
	}

	/* skb frags release userspace buffers */
1240
	for (i = 0; i < num_frags; i++)
1241
		skb_frag_unref(skb, i);
1242 1243

	/* skb frags point to kernel buffers */
1244 1245
	for (i = 0; i < new_frags - 1; i++) {
		__skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1246
		head = (struct page *)page_private(head);
1247
	}
1248 1249
	__skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
	skb_shinfo(skb)->nr_frags = new_frags;
1250

1251
release:
W
Willem de Bruijn 已提交
1252
	skb_zcopy_clear(skb, false);
1253 1254
	return 0;
}
1255
EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1256

H
Herbert Xu 已提交
1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
/**
 *	skb_clone	-	duplicate an sk_buff
 *	@skb: buffer to clone
 *	@gfp_mask: allocation priority
 *
 *	Duplicate an &sk_buff. The new one is not owned by a socket. Both
 *	copies share the same packet data but not structure. The new
 *	buffer has a reference count of 1. If the allocation fails the
 *	function returns %NULL otherwise the new buffer is returned.
 *
 *	If this function is called from an interrupt gfp_mask() must be
 *	%GFP_ATOMIC.
 */

struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
{
1273 1274 1275
	struct sk_buff_fclones *fclones = container_of(skb,
						       struct sk_buff_fclones,
						       skb1);
1276
	struct sk_buff *n;
H
Herbert Xu 已提交
1277

1278 1279
	if (skb_orphan_frags(skb, gfp_mask))
		return NULL;
1280

H
Herbert Xu 已提交
1281
	if (skb->fclone == SKB_FCLONE_ORIG &&
1282
	    refcount_read(&fclones->fclone_ref) == 1) {
1283
		n = &fclones->skb2;
1284
		refcount_set(&fclones->fclone_ref, 2);
H
Herbert Xu 已提交
1285
	} else {
1286 1287 1288
		if (skb_pfmemalloc(skb))
			gfp_mask |= __GFP_MEMALLOC;

H
Herbert Xu 已提交
1289 1290 1291
		n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
		if (!n)
			return NULL;
1292

H
Herbert Xu 已提交
1293 1294 1295 1296
		n->fclone = SKB_FCLONE_UNAVAILABLE;
	}

	return __skb_clone(n, skb);
L
Linus Torvalds 已提交
1297
}
1298
EXPORT_SYMBOL(skb_clone);
L
Linus Torvalds 已提交
1299

1300
void skb_headers_offset_update(struct sk_buff *skb, int off)
1301
{
E
Eric Dumazet 已提交
1302 1303 1304
	/* Only adjust this if it actually is csum_start rather than csum */
	if (skb->ip_summed == CHECKSUM_PARTIAL)
		skb->csum_start += off;
1305 1306 1307 1308 1309 1310 1311
	/* {transport,network,mac}_header and tail are relative to skb->head */
	skb->transport_header += off;
	skb->network_header   += off;
	if (skb_mac_header_was_set(skb))
		skb->mac_header += off;
	skb->inner_transport_header += off;
	skb->inner_network_header += off;
1312
	skb->inner_mac_header += off;
1313
}
1314
EXPORT_SYMBOL(skb_headers_offset_update);
1315

1316
void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
L
Linus Torvalds 已提交
1317
{
1318 1319
	__copy_skb_header(new, old);

1320 1321 1322
	skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
	skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
	skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
L
Linus Torvalds 已提交
1323
}
1324
EXPORT_SYMBOL(skb_copy_header);
L
Linus Torvalds 已提交
1325

1326 1327 1328 1329 1330 1331 1332
static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
{
	if (skb_pfmemalloc(skb))
		return SKB_ALLOC_RX;
	return 0;
}

L
Linus Torvalds 已提交
1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
/**
 *	skb_copy	-	create private copy of an sk_buff
 *	@skb: buffer to copy
 *	@gfp_mask: allocation priority
 *
 *	Make a copy of both an &sk_buff and its data. This is used when the
 *	caller wishes to modify the data and needs a private copy of the
 *	data to alter. Returns %NULL on failure or the pointer to the buffer
 *	on success. The returned buffer has a reference count of 1.
 *
 *	As by-product this function converts non-linear &sk_buff to linear
 *	one, so that &sk_buff becomes completely private and caller is allowed
 *	to modify all the data of returned buffer. This means that this
 *	function is not recommended for use in circumstances when only
 *	header is going to be modified. Use pskb_copy() instead.
 */

A
Al Viro 已提交
1350
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
L
Linus Torvalds 已提交
1351
{
E
Eric Dumazet 已提交
1352
	int headerlen = skb_headroom(skb);
1353
	unsigned int size = skb_end_offset(skb) + skb->data_len;
1354 1355
	struct sk_buff *n = __alloc_skb(size, gfp_mask,
					skb_alloc_rx_flag(skb), NUMA_NO_NODE);
E
Eric Dumazet 已提交
1356

L
Linus Torvalds 已提交
1357 1358 1359 1360 1361 1362 1363 1364
	if (!n)
		return NULL;

	/* Set the data pointer */
	skb_reserve(n, headerlen);
	/* Set the tail pointer and length */
	skb_put(n, skb->len);

1365
	BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
L
Linus Torvalds 已提交
1366

1367
	skb_copy_header(n, skb);
L
Linus Torvalds 已提交
1368 1369
	return n;
}
1370
EXPORT_SYMBOL(skb_copy);
L
Linus Torvalds 已提交
1371 1372

/**
1373
 *	__pskb_copy_fclone	-  create copy of an sk_buff with private head.
L
Linus Torvalds 已提交
1374
 *	@skb: buffer to copy
E
Eric Dumazet 已提交
1375
 *	@headroom: headroom of new skb
L
Linus Torvalds 已提交
1376
 *	@gfp_mask: allocation priority
1377 1378 1379
 *	@fclone: if true allocate the copy of the skb from the fclone
 *	cache instead of the head cache; it is recommended to set this
 *	to true for the cases where the copy will likely be cloned
L
Linus Torvalds 已提交
1380 1381 1382 1383 1384 1385 1386 1387 1388
 *
 *	Make a copy of both an &sk_buff and part of its data, located
 *	in header. Fragmented data remain shared. This is used when
 *	the caller wishes to modify only header of &sk_buff and needs
 *	private copy of the header to alter. Returns %NULL on failure
 *	or the pointer to the buffer on success.
 *	The returned buffer has a reference count of 1.
 */

1389 1390
struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
				   gfp_t gfp_mask, bool fclone)
L
Linus Torvalds 已提交
1391
{
E
Eric Dumazet 已提交
1392
	unsigned int size = skb_headlen(skb) + headroom;
1393 1394
	int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
	struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
E
Eric Dumazet 已提交
1395

L
Linus Torvalds 已提交
1396 1397 1398 1399
	if (!n)
		goto out;

	/* Set the data pointer */
E
Eric Dumazet 已提交
1400
	skb_reserve(n, headroom);
L
Linus Torvalds 已提交
1401 1402 1403
	/* Set the tail pointer and length */
	skb_put(n, skb_headlen(skb));
	/* Copy the bytes */
1404
	skb_copy_from_linear_data(skb, n->data, n->len);
L
Linus Torvalds 已提交
1405

H
Herbert Xu 已提交
1406
	n->truesize += skb->data_len;
L
Linus Torvalds 已提交
1407 1408 1409 1410 1411 1412
	n->data_len  = skb->data_len;
	n->len	     = skb->len;

	if (skb_shinfo(skb)->nr_frags) {
		int i;

W
Willem de Bruijn 已提交
1413 1414
		if (skb_orphan_frags(skb, gfp_mask) ||
		    skb_zerocopy_clone(n, skb, gfp_mask)) {
1415 1416 1417
			kfree_skb(n);
			n = NULL;
			goto out;
1418
		}
L
Linus Torvalds 已提交
1419 1420
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
			skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1421
			skb_frag_ref(skb, i);
L
Linus Torvalds 已提交
1422 1423 1424 1425
		}
		skb_shinfo(n)->nr_frags = i;
	}

1426
	if (skb_has_frag_list(skb)) {
L
Linus Torvalds 已提交
1427 1428 1429 1430
		skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
		skb_clone_fraglist(n);
	}

1431
	skb_copy_header(n, skb);
L
Linus Torvalds 已提交
1432 1433 1434
out:
	return n;
}
1435
EXPORT_SYMBOL(__pskb_copy_fclone);
L
Linus Torvalds 已提交
1436 1437 1438 1439 1440 1441 1442 1443

/**
 *	pskb_expand_head - reallocate header of &sk_buff
 *	@skb: buffer to reallocate
 *	@nhead: room to add at head
 *	@ntail: room to add at tail
 *	@gfp_mask: allocation priority
 *
M
Mathias Krause 已提交
1444 1445
 *	Expands (or creates identical copy, if @nhead and @ntail are zero)
 *	header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
L
Linus Torvalds 已提交
1446 1447 1448 1449 1450 1451 1452
 *	reference count of 1. Returns zero in the case of success or error,
 *	if expansion failed. In the last case, &sk_buff is not changed.
 *
 *	All the pointers pointing into skb header may change and must be
 *	reloaded after call to this function.
 */

V
Victor Fusco 已提交
1453
int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
A
Al Viro 已提交
1454
		     gfp_t gfp_mask)
L
Linus Torvalds 已提交
1455
{
1456 1457
	int i, osize = skb_end_offset(skb);
	int size = osize + nhead + ntail;
L
Linus Torvalds 已提交
1458
	long off;
1459
	u8 *data;
L
Linus Torvalds 已提交
1460

1461 1462
	BUG_ON(nhead < 0);

1463
	BUG_ON(skb_shared(skb));
L
Linus Torvalds 已提交
1464 1465 1466

	size = SKB_DATA_ALIGN(size);

1467 1468 1469 1470
	if (skb_pfmemalloc(skb))
		gfp_mask |= __GFP_MEMALLOC;
	data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
			       gfp_mask, NUMA_NO_NODE, NULL);
L
Linus Torvalds 已提交
1471 1472
	if (!data)
		goto nodata;
1473
	size = SKB_WITH_OVERHEAD(ksize(data));
L
Linus Torvalds 已提交
1474 1475

	/* Copy only real data... and, alas, header. This should be
E
Eric Dumazet 已提交
1476 1477 1478 1479 1480 1481
	 * optimized for the cases when header is void.
	 */
	memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);

	memcpy((struct skb_shared_info *)(data + size),
	       skb_shinfo(skb),
1482
	       offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
L
Linus Torvalds 已提交
1483

1484 1485 1486 1487 1488 1489
	/*
	 * if shinfo is shared we must drop the old head gracefully, but if it
	 * is not we can just drop the old head and let the existing refcount
	 * be since all we did is relocate the values
	 */
	if (skb_cloned(skb)) {
1490 1491
		if (skb_orphan_frags(skb, gfp_mask))
			goto nofrags;
W
Willem de Bruijn 已提交
1492
		if (skb_zcopy(skb))
1493
			refcount_inc(&skb_uarg(skb)->refcnt);
1494
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1495
			skb_frag_ref(skb, i);
L
Linus Torvalds 已提交
1496

1497 1498
		if (skb_has_frag_list(skb))
			skb_clone_fraglist(skb);
L
Linus Torvalds 已提交
1499

1500
		skb_release_data(skb);
1501 1502
	} else {
		skb_free_head(skb);
1503
	}
L
Linus Torvalds 已提交
1504 1505 1506
	off = (data + nhead) - skb->head;

	skb->head     = data;
1507
	skb->head_frag = 0;
L
Linus Torvalds 已提交
1508
	skb->data    += off;
1509 1510
#ifdef NET_SKBUFF_DATA_USES_OFFSET
	skb->end      = size;
1511
	off           = nhead;
1512 1513
#else
	skb->end      = skb->head + size;
1514
#endif
1515
	skb->tail	      += off;
1516
	skb_headers_offset_update(skb, nhead);
L
Linus Torvalds 已提交
1517
	skb->cloned   = 0;
1518
	skb->hdr_len  = 0;
L
Linus Torvalds 已提交
1519 1520
	skb->nohdr    = 0;
	atomic_set(&skb_shinfo(skb)->dataref, 1);
1521

1522 1523
	skb_metadata_clear(skb);

1524 1525 1526 1527 1528 1529 1530
	/* It is not generally safe to change skb->truesize.
	 * For the moment, we really care of rx path, or
	 * when skb is orphaned (not attached to a socket).
	 */
	if (!skb->sk || skb->destructor == sock_edemux)
		skb->truesize += size - osize;

L
Linus Torvalds 已提交
1531 1532
	return 0;

1533 1534
nofrags:
	kfree(data);
L
Linus Torvalds 已提交
1535 1536 1537
nodata:
	return -ENOMEM;
}
1538
EXPORT_SYMBOL(pskb_expand_head);
L
Linus Torvalds 已提交
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558

/* Make private copy of skb with writable head and some headroom */

struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
{
	struct sk_buff *skb2;
	int delta = headroom - skb_headroom(skb);

	if (delta <= 0)
		skb2 = pskb_copy(skb, GFP_ATOMIC);
	else {
		skb2 = skb_clone(skb, GFP_ATOMIC);
		if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
					     GFP_ATOMIC)) {
			kfree_skb(skb2);
			skb2 = NULL;
		}
	}
	return skb2;
}
1559
EXPORT_SYMBOL(skb_realloc_headroom);
L
Linus Torvalds 已提交
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579

/**
 *	skb_copy_expand	-	copy and expand sk_buff
 *	@skb: buffer to copy
 *	@newheadroom: new free bytes at head
 *	@newtailroom: new free bytes at tail
 *	@gfp_mask: allocation priority
 *
 *	Make a copy of both an &sk_buff and its data and while doing so
 *	allocate additional space.
 *
 *	This is used when the caller wishes to modify the data and needs a
 *	private copy of the data to alter as well as more space for new fields.
 *	Returns %NULL on failure or the pointer to the buffer
 *	on success. The returned buffer has a reference count of 1.
 *
 *	You must pass %GFP_ATOMIC as the allocation priority if this function
 *	is called from an interrupt.
 */
struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
V
Victor Fusco 已提交
1580
				int newheadroom, int newtailroom,
A
Al Viro 已提交
1581
				gfp_t gfp_mask)
L
Linus Torvalds 已提交
1582 1583 1584 1585
{
	/*
	 *	Allocate the copy buffer
	 */
1586 1587 1588
	struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
					gfp_mask, skb_alloc_rx_flag(skb),
					NUMA_NO_NODE);
1589
	int oldheadroom = skb_headroom(skb);
L
Linus Torvalds 已提交
1590 1591 1592 1593 1594 1595 1596 1597 1598 1599
	int head_copy_len, head_copy_off;

	if (!n)
		return NULL;

	skb_reserve(n, newheadroom);

	/* Set the tail pointer and length */
	skb_put(n, skb->len);

1600
	head_copy_len = oldheadroom;
L
Linus Torvalds 已提交
1601 1602 1603 1604 1605 1606 1607
	head_copy_off = 0;
	if (newheadroom <= head_copy_len)
		head_copy_len = newheadroom;
	else
		head_copy_off = newheadroom - head_copy_len;

	/* Copy the linear header and data. */
1608 1609
	BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
			     skb->len + head_copy_len));
L
Linus Torvalds 已提交
1610

1611
	skb_copy_header(n, skb);
L
Linus Torvalds 已提交
1612

E
Eric Dumazet 已提交
1613
	skb_headers_offset_update(n, newheadroom - oldheadroom);
1614

L
Linus Torvalds 已提交
1615 1616
	return n;
}
1617
EXPORT_SYMBOL(skb_copy_expand);
L
Linus Torvalds 已提交
1618 1619

/**
1620
 *	__skb_pad		-	zero pad the tail of an skb
L
Linus Torvalds 已提交
1621 1622
 *	@skb: buffer to pad
 *	@pad: space to pad
1623
 *	@free_on_error: free buffer on error
L
Linus Torvalds 已提交
1624 1625 1626 1627 1628
 *
 *	Ensure that a buffer is followed by a padding area that is zero
 *	filled. Used by network drivers which may DMA or transfer data
 *	beyond the buffer end onto the wire.
 *
1629 1630
 *	May return error in out of memory cases. The skb is freed on error
 *	if @free_on_error is true.
L
Linus Torvalds 已提交
1631
 */
1632

1633
int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
L
Linus Torvalds 已提交
1634
{
1635 1636
	int err;
	int ntail;
1637

L
Linus Torvalds 已提交
1638
	/* If the skbuff is non linear tailroom is always zero.. */
1639
	if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
L
Linus Torvalds 已提交
1640
		memset(skb->data+skb->len, 0, pad);
1641
		return 0;
L
Linus Torvalds 已提交
1642
	}
1643

1644
	ntail = skb->data_len + pad - (skb->end - skb->tail);
1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
	if (likely(skb_cloned(skb) || ntail > 0)) {
		err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
		if (unlikely(err))
			goto free_skb;
	}

	/* FIXME: The use of this function with non-linear skb's really needs
	 * to be audited.
	 */
	err = skb_linearize(skb);
	if (unlikely(err))
		goto free_skb;

	memset(skb->data + skb->len, 0, pad);
	return 0;

free_skb:
1662 1663
	if (free_on_error)
		kfree_skb(skb);
1664
	return err;
1665
}
1666
EXPORT_SYMBOL(__skb_pad);
1667

M
Mathias Krause 已提交
1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
/**
 *	pskb_put - add data to the tail of a potentially fragmented buffer
 *	@skb: start of the buffer to use
 *	@tail: tail fragment of the buffer to use
 *	@len: amount of data to add
 *
 *	This function extends the used data area of the potentially
 *	fragmented buffer. @tail must be the last fragment of @skb -- or
 *	@skb itself. If this would exceed the total buffer size the kernel
 *	will panic. A pointer to the first byte of the extra data is
 *	returned.
 */

1681
void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
M
Mathias Krause 已提交
1682 1683 1684 1685 1686 1687 1688 1689 1690
{
	if (tail != skb) {
		skb->data_len += len;
		skb->len += len;
	}
	return skb_put(tail, len);
}
EXPORT_SYMBOL_GPL(pskb_put);

1691 1692 1693 1694 1695 1696 1697 1698 1699
/**
 *	skb_put - add data to a buffer
 *	@skb: buffer to use
 *	@len: amount of data to add
 *
 *	This function extends the used data area of the buffer. If this would
 *	exceed the total buffer size the kernel will panic. A pointer to the
 *	first byte of the extra data is returned.
 */
1700
void *skb_put(struct sk_buff *skb, unsigned int len)
1701
{
1702
	void *tmp = skb_tail_pointer(skb);
1703 1704 1705 1706 1707 1708 1709 1710 1711
	SKB_LINEAR_ASSERT(skb);
	skb->tail += len;
	skb->len  += len;
	if (unlikely(skb->tail > skb->end))
		skb_over_panic(skb, len, __builtin_return_address(0));
	return tmp;
}
EXPORT_SYMBOL(skb_put);

1712 1713 1714 1715 1716 1717 1718 1719 1720
/**
 *	skb_push - add data to the start of a buffer
 *	@skb: buffer to use
 *	@len: amount of data to add
 *
 *	This function extends the used data area of the buffer at the buffer
 *	start. If this would exceed the total buffer headroom the kernel will
 *	panic. A pointer to the first byte of the extra data is returned.
 */
1721
void *skb_push(struct sk_buff *skb, unsigned int len)
1722 1723 1724
{
	skb->data -= len;
	skb->len  += len;
1725
	if (unlikely(skb->data < skb->head))
1726 1727 1728 1729 1730
		skb_under_panic(skb, len, __builtin_return_address(0));
	return skb->data;
}
EXPORT_SYMBOL(skb_push);

1731 1732 1733 1734 1735 1736 1737 1738 1739 1740
/**
 *	skb_pull - remove data from the start of a buffer
 *	@skb: buffer to use
 *	@len: amount of data to remove
 *
 *	This function removes data from the start of a buffer, returning
 *	the memory to the headroom. A pointer to the next data in the buffer
 *	is returned. Once the data has been pulled future pushes will overwrite
 *	the old data.
 */
1741
void *skb_pull(struct sk_buff *skb, unsigned int len)
1742
{
1743
	return skb_pull_inline(skb, len);
1744 1745 1746
}
EXPORT_SYMBOL(skb_pull);

1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762
/**
 *	skb_trim - remove end from a buffer
 *	@skb: buffer to alter
 *	@len: new length
 *
 *	Cut the length of a buffer down by removing data from the tail. If
 *	the buffer is already under the length specified it is not modified.
 *	The skb must be linear.
 */
void skb_trim(struct sk_buff *skb, unsigned int len)
{
	if (skb->len > len)
		__skb_trim(skb, len);
}
EXPORT_SYMBOL(skb_trim);

1763
/* Trims skb to length len. It can change skb pointers.
L
Linus Torvalds 已提交
1764 1765
 */

1766
int ___pskb_trim(struct sk_buff *skb, unsigned int len)
L
Linus Torvalds 已提交
1767
{
1768 1769
	struct sk_buff **fragp;
	struct sk_buff *frag;
L
Linus Torvalds 已提交
1770 1771 1772
	int offset = skb_headlen(skb);
	int nfrags = skb_shinfo(skb)->nr_frags;
	int i;
1773 1774 1775 1776 1777
	int err;

	if (skb_cloned(skb) &&
	    unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
		return err;
L
Linus Torvalds 已提交
1778

1779 1780 1781 1782 1783
	i = 0;
	if (offset >= len)
		goto drop_pages;

	for (; i < nfrags; i++) {
E
Eric Dumazet 已提交
1784
		int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1785 1786 1787 1788 1789 1790

		if (end < len) {
			offset = end;
			continue;
		}

E
Eric Dumazet 已提交
1791
		skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1792

1793
drop_pages:
1794 1795 1796
		skb_shinfo(skb)->nr_frags = i;

		for (; i < nfrags; i++)
1797
			skb_frag_unref(skb, i);
1798

1799
		if (skb_has_frag_list(skb))
1800
			skb_drop_fraglist(skb);
1801
		goto done;
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815
	}

	for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
	     fragp = &frag->next) {
		int end = offset + frag->len;

		if (skb_shared(frag)) {
			struct sk_buff *nfrag;

			nfrag = skb_clone(frag, GFP_ATOMIC);
			if (unlikely(!nfrag))
				return -ENOMEM;

			nfrag->next = frag->next;
1816
			consume_skb(frag);
1817 1818
			frag = nfrag;
			*fragp = frag;
L
Linus Torvalds 已提交
1819
		}
1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832

		if (end < len) {
			offset = end;
			continue;
		}

		if (end > len &&
		    unlikely((err = pskb_trim(frag, len - offset))))
			return err;

		if (frag->next)
			skb_drop_list(&frag->next);
		break;
L
Linus Torvalds 已提交
1833 1834
	}

1835
done:
1836
	if (len > skb_headlen(skb)) {
L
Linus Torvalds 已提交
1837 1838 1839
		skb->data_len -= skb->len - len;
		skb->len       = len;
	} else {
1840 1841
		skb->len       = len;
		skb->data_len  = 0;
1842
		skb_set_tail_pointer(skb, len);
L
Linus Torvalds 已提交
1843 1844
	}

1845 1846
	if (!skb->sk || skb->destructor == sock_edemux)
		skb_condense(skb);
L
Linus Torvalds 已提交
1847 1848
	return 0;
}
1849
EXPORT_SYMBOL(___pskb_trim);
L
Linus Torvalds 已提交
1850

1851 1852 1853 1854 1855 1856 1857
/* Note : use pskb_trim_rcsum() instead of calling this directly
 */
int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
{
	if (skb->ip_summed == CHECKSUM_COMPLETE) {
		int delta = skb->len - len;

1858 1859 1860
		skb->csum = csum_block_sub(skb->csum,
					   skb_checksum(skb, len, delta, 0),
					   len);
1861 1862 1863 1864 1865
	}
	return __pskb_trim(skb, len);
}
EXPORT_SYMBOL(pskb_trim_rcsum_slow);

L
Linus Torvalds 已提交
1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890
/**
 *	__pskb_pull_tail - advance tail of skb header
 *	@skb: buffer to reallocate
 *	@delta: number of bytes to advance tail
 *
 *	The function makes a sense only on a fragmented &sk_buff,
 *	it expands header moving its tail forward and copying necessary
 *	data from fragmented part.
 *
 *	&sk_buff MUST have reference count of 1.
 *
 *	Returns %NULL (and &sk_buff does not change) if pull failed
 *	or value of new tail of skb in the case of success.
 *
 *	All the pointers pointing into skb header may change and must be
 *	reloaded after call to this function.
 */

/* Moves tail of skb head forward, copying data from fragmented part,
 * when it is necessary.
 * 1. It may fail due to malloc failure.
 * 2. It may change skb pointers.
 *
 * It is pretty complicated. Luckily, it is called only in exceptional cases.
 */
1891
void *__pskb_pull_tail(struct sk_buff *skb, int delta)
L
Linus Torvalds 已提交
1892 1893 1894 1895 1896
{
	/* If skb has not enough free space at tail, get new one
	 * plus 128 bytes for future expansions. If we have enough
	 * room at tail, reallocate without expansion only if skb is cloned.
	 */
1897
	int i, k, eat = (skb->tail + delta) - skb->end;
L
Linus Torvalds 已提交
1898 1899 1900 1901 1902 1903 1904

	if (eat > 0 || skb_cloned(skb)) {
		if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
				     GFP_ATOMIC))
			return NULL;
	}

1905 1906
	BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
			     skb_tail_pointer(skb), delta));
L
Linus Torvalds 已提交
1907 1908 1909 1910

	/* Optimization: no fragments, no reasons to preestimate
	 * size of pulled pages. Superb.
	 */
1911
	if (!skb_has_frag_list(skb))
L
Linus Torvalds 已提交
1912 1913 1914 1915 1916
		goto pull_pages;

	/* Estimate size of pulled pages. */
	eat = delta;
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
E
Eric Dumazet 已提交
1917 1918 1919
		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);

		if (size >= eat)
L
Linus Torvalds 已提交
1920
			goto pull_pages;
E
Eric Dumazet 已提交
1921
		eat -= size;
L
Linus Torvalds 已提交
1922 1923 1924
	}

	/* If we need update frag list, we are in troubles.
W
Wenhua Shi 已提交
1925
	 * Certainly, it is possible to add an offset to skb data,
L
Linus Torvalds 已提交
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957
	 * but taking into account that pulling is expected to
	 * be very rare operation, it is worth to fight against
	 * further bloating skb head and crucify ourselves here instead.
	 * Pure masohism, indeed. 8)8)
	 */
	if (eat) {
		struct sk_buff *list = skb_shinfo(skb)->frag_list;
		struct sk_buff *clone = NULL;
		struct sk_buff *insp = NULL;

		do {
			if (list->len <= eat) {
				/* Eaten as whole. */
				eat -= list->len;
				list = list->next;
				insp = list;
			} else {
				/* Eaten partially. */

				if (skb_shared(list)) {
					/* Sucks! We need to fork list. :-( */
					clone = skb_clone(list, GFP_ATOMIC);
					if (!clone)
						return NULL;
					insp = list->next;
					list = clone;
				} else {
					/* This may be pulled without
					 * problems. */
					insp = list;
				}
				if (!pskb_pull(list, eat)) {
1958
					kfree_skb(clone);
L
Linus Torvalds 已提交
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981
					return NULL;
				}
				break;
			}
		} while (eat);

		/* Free pulled out fragments. */
		while ((list = skb_shinfo(skb)->frag_list) != insp) {
			skb_shinfo(skb)->frag_list = list->next;
			kfree_skb(list);
		}
		/* And insert new clone at head. */
		if (clone) {
			clone->next = list;
			skb_shinfo(skb)->frag_list = clone;
		}
	}
	/* Success! Now we may commit changes to skb data. */

pull_pages:
	eat = delta;
	k = 0;
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
E
Eric Dumazet 已提交
1982 1983 1984
		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);

		if (size <= eat) {
1985
			skb_frag_unref(skb, i);
E
Eric Dumazet 已提交
1986
			eat -= size;
L
Linus Torvalds 已提交
1987 1988 1989 1990
		} else {
			skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
			if (eat) {
				skb_shinfo(skb)->frags[k].page_offset += eat;
E
Eric Dumazet 已提交
1991
				skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1992 1993
				if (!i)
					goto end;
L
Linus Torvalds 已提交
1994 1995 1996 1997 1998 1999 2000
				eat = 0;
			}
			k++;
		}
	}
	skb_shinfo(skb)->nr_frags = k;

2001
end:
L
Linus Torvalds 已提交
2002 2003 2004
	skb->tail     += delta;
	skb->data_len -= delta;

W
Willem de Bruijn 已提交
2005 2006 2007
	if (!skb->data_len)
		skb_zcopy_clear(skb, false);

2008
	return skb_tail_pointer(skb);
L
Linus Torvalds 已提交
2009
}
2010
EXPORT_SYMBOL(__pskb_pull_tail);
L
Linus Torvalds 已提交
2011

2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
/**
 *	skb_copy_bits - copy bits from skb to kernel buffer
 *	@skb: source skb
 *	@offset: offset in source
 *	@to: destination buffer
 *	@len: number of bytes to copy
 *
 *	Copy the specified number of bytes from the source skb to the
 *	destination buffer.
 *
 *	CAUTION ! :
 *		If its prototype is ever changed,
 *		check arch/{*}/net/{*}.S files,
 *		since it is called from BPF assembly code.
 */
L
Linus Torvalds 已提交
2027 2028
int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
{
2029
	int start = skb_headlen(skb);
2030 2031
	struct sk_buff *frag_iter;
	int i, copy;
L
Linus Torvalds 已提交
2032 2033 2034 2035 2036

	if (offset > (int)skb->len - len)
		goto fault;

	/* Copy header. */
2037
	if ((copy = start - offset) > 0) {
L
Linus Torvalds 已提交
2038 2039
		if (copy > len)
			copy = len;
2040
		skb_copy_from_linear_data_offset(skb, offset, to, copy);
L
Linus Torvalds 已提交
2041 2042 2043 2044 2045 2046 2047
		if ((len -= copy) == 0)
			return 0;
		offset += copy;
		to     += copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2048
		int end;
E
Eric Dumazet 已提交
2049
		skb_frag_t *f = &skb_shinfo(skb)->frags[i];
L
Linus Torvalds 已提交
2050

2051
		WARN_ON(start > offset + len);
2052

E
Eric Dumazet 已提交
2053
		end = start + skb_frag_size(f);
L
Linus Torvalds 已提交
2054
		if ((copy = end - offset) > 0) {
2055 2056
			u32 p_off, p_len, copied;
			struct page *p;
L
Linus Torvalds 已提交
2057 2058 2059 2060 2061
			u8 *vaddr;

			if (copy > len)
				copy = len;

2062 2063 2064 2065 2066 2067 2068
			skb_frag_foreach_page(f,
					      f->page_offset + offset - start,
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
				memcpy(to + copied, vaddr + p_off, p_len);
				kunmap_atomic(vaddr);
			}
L
Linus Torvalds 已提交
2069 2070 2071 2072 2073 2074

			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			to     += copy;
		}
2075
		start = end;
L
Linus Torvalds 已提交
2076 2077
	}

2078 2079
	skb_walk_frags(skb, frag_iter) {
		int end;
L
Linus Torvalds 已提交
2080

2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			if (copy > len)
				copy = len;
			if (skb_copy_bits(frag_iter, offset - start, to, copy))
				goto fault;
			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			to     += copy;
L
Linus Torvalds 已提交
2093
		}
2094
		start = end;
L
Linus Torvalds 已提交
2095
	}
2096

L
Linus Torvalds 已提交
2097 2098 2099 2100 2101 2102
	if (!len)
		return 0;

fault:
	return -EFAULT;
}
2103
EXPORT_SYMBOL(skb_copy_bits);
L
Linus Torvalds 已提交
2104

J
Jens Axboe 已提交
2105 2106 2107 2108 2109 2110
/*
 * Callback from splice_to_pipe(), if we need to release some pages
 * at the end of the spd in case we error'ed out in filling the pipe.
 */
static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
2111 2112
	put_page(spd->pages[i]);
}
J
Jens Axboe 已提交
2113

2114 2115
static struct page *linear_to_page(struct page *page, unsigned int *len,
				   unsigned int *offset,
2116
				   struct sock *sk)
2117
{
2118
	struct page_frag *pfrag = sk_page_frag(sk);
2119

2120 2121
	if (!sk_page_frag_refill(sk, pfrag))
		return NULL;
2122

2123
	*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2124

2125 2126 2127 2128
	memcpy(page_address(pfrag->page) + pfrag->offset,
	       page_address(page) + *offset, *len);
	*offset = pfrag->offset;
	pfrag->offset += *len;
2129

2130
	return pfrag->page;
J
Jens Axboe 已提交
2131 2132
}

E
Eric Dumazet 已提交
2133 2134 2135 2136 2137 2138 2139 2140 2141 2142
static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
			     struct page *page,
			     unsigned int offset)
{
	return	spd->nr_pages &&
		spd->pages[spd->nr_pages - 1] == page &&
		(spd->partial[spd->nr_pages - 1].offset +
		 spd->partial[spd->nr_pages - 1].len == offset);
}

J
Jens Axboe 已提交
2143 2144 2145
/*
 * Fill page/offset/length into spd, if it can hold more pages.
 */
2146 2147 2148
static bool spd_fill_page(struct splice_pipe_desc *spd,
			  struct pipe_inode_info *pipe, struct page *page,
			  unsigned int *len, unsigned int offset,
2149
			  bool linear,
2150
			  struct sock *sk)
J
Jens Axboe 已提交
2151
{
E
Eric Dumazet 已提交
2152
	if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2153
		return true;
J
Jens Axboe 已提交
2154

2155
	if (linear) {
2156
		page = linear_to_page(page, len, &offset, sk);
2157
		if (!page)
2158
			return true;
E
Eric Dumazet 已提交
2159 2160 2161
	}
	if (spd_can_coalesce(spd, page, offset)) {
		spd->partial[spd->nr_pages - 1].len += *len;
2162
		return false;
E
Eric Dumazet 已提交
2163 2164
	}
	get_page(page);
J
Jens Axboe 已提交
2165
	spd->pages[spd->nr_pages] = page;
2166
	spd->partial[spd->nr_pages].len = *len;
J
Jens Axboe 已提交
2167 2168
	spd->partial[spd->nr_pages].offset = offset;
	spd->nr_pages++;
2169

2170
	return false;
J
Jens Axboe 已提交
2171 2172
}

2173 2174
static bool __splice_segment(struct page *page, unsigned int poff,
			     unsigned int plen, unsigned int *off,
2175
			     unsigned int *len,
2176
			     struct splice_pipe_desc *spd, bool linear,
2177 2178
			     struct sock *sk,
			     struct pipe_inode_info *pipe)
J
Jens Axboe 已提交
2179
{
2180
	if (!*len)
2181
		return true;
2182 2183 2184 2185

	/* skip this segment if already processed */
	if (*off >= plen) {
		*off -= plen;
2186
		return false;
2187
	}
J
Jens Axboe 已提交
2188

2189
	/* ignore any bits we already processed */
2190 2191 2192
	poff += *off;
	plen -= *off;
	*off = 0;
J
Jens Axboe 已提交
2193

2194 2195
	do {
		unsigned int flen = min(*len, plen);
2196

2197 2198 2199 2200 2201 2202 2203
		if (spd_fill_page(spd, pipe, page, &flen, poff,
				  linear, sk))
			return true;
		poff += flen;
		plen -= flen;
		*len -= flen;
	} while (*len && plen);
2204

2205
	return false;
2206 2207 2208
}

/*
2209
 * Map linear and fragment data from the skb to spd. It reports true if the
2210 2211
 * pipe is full or if we already spliced the requested length.
 */
2212 2213 2214
static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
			      unsigned int *offset, unsigned int *len,
			      struct splice_pipe_desc *spd, struct sock *sk)
2215 2216
{
	int seg;
2217
	struct sk_buff *iter;
2218

2219
	/* map the linear part :
2220 2221 2222
	 * If skb->head_frag is set, this 'linear' part is backed by a
	 * fragment, and if the head is not shared with any clones then
	 * we can avoid a copy since we own the head portion of this page.
2223 2224 2225 2226
	 */
	if (__splice_segment(virt_to_page(skb->data),
			     (unsigned long) skb->data & (PAGE_SIZE - 1),
			     skb_headlen(skb),
2227
			     offset, len, spd,
2228
			     skb_head_is_locked(skb),
2229
			     sk, pipe))
2230
		return true;
J
Jens Axboe 已提交
2231 2232 2233 2234 2235 2236 2237

	/*
	 * then map the fragments
	 */
	for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
		const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];

2238
		if (__splice_segment(skb_frag_page(f),
E
Eric Dumazet 已提交
2239
				     f->page_offset, skb_frag_size(f),
2240
				     offset, len, spd, false, sk, pipe))
2241
			return true;
J
Jens Axboe 已提交
2242 2243
	}

2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
	skb_walk_frags(skb, iter) {
		if (*offset >= iter->len) {
			*offset -= iter->len;
			continue;
		}
		/* __skb_splice_bits() only fails if the output has no room
		 * left, so no point in going over the frag_list for the error
		 * case.
		 */
		if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
			return true;
	}

2257
	return false;
J
Jens Axboe 已提交
2258 2259 2260 2261
}

/*
 * Map data from the skb to a pipe. Should handle both the linear part,
2262
 * the fragments, and the frag list.
J
Jens Axboe 已提交
2263
 */
2264
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
J
Jens Axboe 已提交
2265
		    struct pipe_inode_info *pipe, unsigned int tlen,
A
Al Viro 已提交
2266
		    unsigned int flags)
J
Jens Axboe 已提交
2267
{
E
Eric Dumazet 已提交
2268 2269
	struct partial_page partial[MAX_SKB_FRAGS];
	struct page *pages[MAX_SKB_FRAGS];
J
Jens Axboe 已提交
2270 2271 2272
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
2273
		.nr_pages_max = MAX_SKB_FRAGS,
M
Miklos Szeredi 已提交
2274
		.ops = &nosteal_pipe_buf_ops,
J
Jens Axboe 已提交
2275 2276
		.spd_release = sock_spd_release,
	};
2277 2278
	int ret = 0;

2279
	__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
J
Jens Axboe 已提交
2280

2281
	if (spd.nr_pages)
A
Al Viro 已提交
2282
		ret = splice_to_pipe(pipe, &spd);
J
Jens Axboe 已提交
2283

2284
	return ret;
J
Jens Axboe 已提交
2285
}
2286
EXPORT_SYMBOL_GPL(skb_splice_bits);
J
Jens Axboe 已提交
2287

2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305
/* Send skb data on a socket. Socket must be locked. */
int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
			 int len)
{
	unsigned int orig_len = len;
	struct sk_buff *head = skb;
	unsigned short fragidx;
	int slen, ret;

do_frag_list:

	/* Deal with head data */
	while (offset < skb_headlen(skb) && len) {
		struct kvec kv;
		struct msghdr msg;

		slen = min_t(int, len, skb_headlen(skb) - offset);
		kv.iov_base = skb->data + offset;
J
John Fastabend 已提交
2306
		kv.iov_len = slen;
2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
		memset(&msg, 0, sizeof(msg));

		ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
		if (ret <= 0)
			goto error;

		offset += ret;
		len -= ret;
	}

	/* All the data was skb head? */
	if (!len)
		goto out;

	/* Make offset relative to start of frags */
	offset -= skb_headlen(skb);

	/* Find where we are in frag list */
	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];

		if (offset < frag->size)
			break;

		offset -= frag->size;
	}

	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];

		slen = min_t(size_t, len, frag->size - offset);

		while (slen) {
			ret = kernel_sendpage_locked(sk, frag->page.p,
						     frag->page_offset + offset,
						     slen, MSG_DONTWAIT);
			if (ret <= 0)
				goto error;

			len -= ret;
			offset += ret;
			slen -= ret;
		}

		offset = 0;
	}

	if (len) {
		/* Process any frag lists */

		if (skb == head) {
			if (skb_has_frag_list(skb)) {
				skb = skb_shinfo(skb)->frag_list;
				goto do_frag_list;
			}
		} else if (skb->next) {
			skb = skb->next;
			goto do_frag_list;
		}
	}

out:
	return orig_len - len;

error:
	return orig_len == len ? ret : orig_len - len;
}
EXPORT_SYMBOL_GPL(skb_send_sock_locked);

2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387
/**
 *	skb_store_bits - store bits from kernel buffer to skb
 *	@skb: destination buffer
 *	@offset: offset in destination
 *	@from: source buffer
 *	@len: number of bytes to copy
 *
 *	Copy the specified number of bytes from the source buffer to the
 *	destination skb.  This function handles all the messy bits of
 *	traversing fragment lists and such.
 */

2388
int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2389
{
2390
	int start = skb_headlen(skb);
2391 2392
	struct sk_buff *frag_iter;
	int i, copy;
2393 2394 2395 2396

	if (offset > (int)skb->len - len)
		goto fault;

2397
	if ((copy = start - offset) > 0) {
2398 2399
		if (copy > len)
			copy = len;
2400
		skb_copy_to_linear_data_offset(skb, offset, from, copy);
2401 2402 2403 2404 2405 2406 2407 2408
		if ((len -= copy) == 0)
			return 0;
		offset += copy;
		from += copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2409 2410
		int end;

2411
		WARN_ON(start > offset + len);
2412

E
Eric Dumazet 已提交
2413
		end = start + skb_frag_size(frag);
2414
		if ((copy = end - offset) > 0) {
2415 2416
			u32 p_off, p_len, copied;
			struct page *p;
2417 2418 2419 2420 2421
			u8 *vaddr;

			if (copy > len)
				copy = len;

2422 2423 2424 2425 2426 2427 2428
			skb_frag_foreach_page(frag,
					      frag->page_offset + offset - start,
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
				memcpy(vaddr + p_off, from + copied, p_len);
				kunmap_atomic(vaddr);
			}
2429 2430 2431 2432 2433 2434

			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			from += copy;
		}
2435
		start = end;
2436 2437
	}

2438 2439
	skb_walk_frags(skb, frag_iter) {
		int end;
2440

2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			if (copy > len)
				copy = len;
			if (skb_store_bits(frag_iter, offset - start,
					   from, copy))
				goto fault;
			if ((len -= copy) == 0)
				return 0;
			offset += copy;
			from += copy;
2454
		}
2455
		start = end;
2456 2457 2458 2459 2460 2461 2462 2463 2464
	}
	if (!len)
		return 0;

fault:
	return -EFAULT;
}
EXPORT_SYMBOL(skb_store_bits);

L
Linus Torvalds 已提交
2465
/* Checksum skb data. */
2466 2467
__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
		      __wsum csum, const struct skb_checksum_ops *ops)
L
Linus Torvalds 已提交
2468
{
2469 2470
	int start = skb_headlen(skb);
	int i, copy = start - offset;
2471
	struct sk_buff *frag_iter;
L
Linus Torvalds 已提交
2472 2473 2474 2475 2476 2477
	int pos = 0;

	/* Checksum header. */
	if (copy > 0) {
		if (copy > len)
			copy = len;
2478
		csum = ops->update(skb->data + offset, copy, csum);
L
Linus Torvalds 已提交
2479 2480 2481 2482 2483 2484 2485
		if ((len -= copy) == 0)
			return csum;
		offset += copy;
		pos	= copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2486
		int end;
E
Eric Dumazet 已提交
2487
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2488

2489
		WARN_ON(start > offset + len);
L
Linus Torvalds 已提交
2490

E
Eric Dumazet 已提交
2491
		end = start + skb_frag_size(frag);
L
Linus Torvalds 已提交
2492
		if ((copy = end - offset) > 0) {
2493 2494
			u32 p_off, p_len, copied;
			struct page *p;
2495
			__wsum csum2;
L
Linus Torvalds 已提交
2496 2497 2498 2499
			u8 *vaddr;

			if (copy > len)
				copy = len;
2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510

			skb_frag_foreach_page(frag,
					      frag->page_offset + offset - start,
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
				csum2 = ops->update(vaddr + p_off, p_len, 0);
				kunmap_atomic(vaddr);
				csum = ops->combine(csum, csum2, pos, p_len);
				pos += p_len;
			}

L
Linus Torvalds 已提交
2511 2512 2513 2514
			if (!(len -= copy))
				return csum;
			offset += copy;
		}
2515
		start = end;
L
Linus Torvalds 已提交
2516 2517
	}

2518 2519
	skb_walk_frags(skb, frag_iter) {
		int end;
L
Linus Torvalds 已提交
2520

2521 2522 2523 2524 2525 2526 2527
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			__wsum csum2;
			if (copy > len)
				copy = len;
2528 2529 2530
			csum2 = __skb_checksum(frag_iter, offset - start,
					       copy, 0, ops);
			csum = ops->combine(csum, csum2, pos, copy);
2531 2532 2533 2534
			if ((len -= copy) == 0)
				return csum;
			offset += copy;
			pos    += copy;
L
Linus Torvalds 已提交
2535
		}
2536
		start = end;
L
Linus Torvalds 已提交
2537
	}
2538
	BUG_ON(len);
L
Linus Torvalds 已提交
2539 2540 2541

	return csum;
}
2542 2543 2544 2545 2546 2547
EXPORT_SYMBOL(__skb_checksum);

__wsum skb_checksum(const struct sk_buff *skb, int offset,
		    int len, __wsum csum)
{
	const struct skb_checksum_ops ops = {
2548
		.update  = csum_partial_ext,
2549 2550 2551 2552 2553
		.combine = csum_block_add_ext,
	};

	return __skb_checksum(skb, offset, len, csum, &ops);
}
2554
EXPORT_SYMBOL(skb_checksum);
L
Linus Torvalds 已提交
2555 2556 2557

/* Both of above in one bottle. */

2558 2559
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
				    u8 *to, int len, __wsum csum)
L
Linus Torvalds 已提交
2560
{
2561 2562
	int start = skb_headlen(skb);
	int i, copy = start - offset;
2563
	struct sk_buff *frag_iter;
L
Linus Torvalds 已提交
2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
	int pos = 0;

	/* Copy header. */
	if (copy > 0) {
		if (copy > len)
			copy = len;
		csum = csum_partial_copy_nocheck(skb->data + offset, to,
						 copy, csum);
		if ((len -= copy) == 0)
			return csum;
		offset += copy;
		to     += copy;
		pos	= copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2580 2581
		int end;

2582
		WARN_ON(start > offset + len);
L
Linus Torvalds 已提交
2583

E
Eric Dumazet 已提交
2584
		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
L
Linus Torvalds 已提交
2585
		if ((copy = end - offset) > 0) {
2586 2587 2588
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
			u32 p_off, p_len, copied;
			struct page *p;
2589
			__wsum csum2;
L
Linus Torvalds 已提交
2590 2591 2592 2593
			u8 *vaddr;

			if (copy > len)
				copy = len;
2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606

			skb_frag_foreach_page(frag,
					      frag->page_offset + offset - start,
					      copy, p, p_off, p_len, copied) {
				vaddr = kmap_atomic(p);
				csum2 = csum_partial_copy_nocheck(vaddr + p_off,
								  to + copied,
								  p_len, 0);
				kunmap_atomic(vaddr);
				csum = csum_block_add(csum, csum2, pos);
				pos += p_len;
			}

L
Linus Torvalds 已提交
2607 2608 2609 2610 2611
			if (!(len -= copy))
				return csum;
			offset += copy;
			to     += copy;
		}
2612
		start = end;
L
Linus Torvalds 已提交
2613 2614
	}

2615 2616 2617
	skb_walk_frags(skb, frag_iter) {
		__wsum csum2;
		int end;
L
Linus Torvalds 已提交
2618

2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633
		WARN_ON(start > offset + len);

		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
			if (copy > len)
				copy = len;
			csum2 = skb_copy_and_csum_bits(frag_iter,
						       offset - start,
						       to, copy, 0);
			csum = csum_block_add(csum, csum2, pos);
			if ((len -= copy) == 0)
				return csum;
			offset += copy;
			to     += copy;
			pos    += copy;
L
Linus Torvalds 已提交
2634
		}
2635
		start = end;
L
Linus Torvalds 已提交
2636
	}
2637
	BUG_ON(len);
L
Linus Torvalds 已提交
2638 2639
	return csum;
}
2640
EXPORT_SYMBOL(skb_copy_and_csum_bits);
L
Linus Torvalds 已提交
2641

2642 2643 2644 2645 2646
__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
{
	__sum16 sum;

	sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
2647
	/* See comments in __skb_checksum_complete(). */
2648 2649 2650
	if (likely(!sum)) {
		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
		    !skb->csum_complete_sw)
2651
			netdev_rx_csum_fault(skb->dev, skb);
2652 2653 2654 2655 2656 2657 2658
	}
	if (!skb_shared(skb))
		skb->csum_valid = !sum;
	return sum;
}
EXPORT_SYMBOL(__skb_checksum_complete_head);

2659 2660 2661 2662 2663 2664 2665 2666 2667
/* This function assumes skb->csum already holds pseudo header's checksum,
 * which has been changed from the hardware checksum, for example, by
 * __skb_checksum_validate_complete(). And, the original skb->csum must
 * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
 *
 * It returns non-zero if the recomputed checksum is still invalid, otherwise
 * zero. The new checksum is stored back into skb->csum unless the skb is
 * shared.
 */
2668 2669 2670 2671 2672 2673 2674 2675
__sum16 __skb_checksum_complete(struct sk_buff *skb)
{
	__wsum csum;
	__sum16 sum;

	csum = skb_checksum(skb, 0, skb->len, 0);

	sum = csum_fold(csum_add(skb->csum, csum));
2676 2677 2678 2679 2680 2681 2682
	/* This check is inverted, because we already knew the hardware
	 * checksum is invalid before calling this function. So, if the
	 * re-computed checksum is valid instead, then we have a mismatch
	 * between the original skb->csum and skb_checksum(). This means either
	 * the original hardware checksum is incorrect or we screw up skb->csum
	 * when moving skb->data around.
	 */
2683 2684 2685
	if (likely(!sum)) {
		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
		    !skb->csum_complete_sw)
2686
			netdev_rx_csum_fault(skb->dev, skb);
2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700
	}

	if (!skb_shared(skb)) {
		/* Save full packet checksum */
		skb->csum = csum;
		skb->ip_summed = CHECKSUM_COMPLETE;
		skb->csum_complete_sw = 1;
		skb->csum_valid = !sum;
	}

	return sum;
}
EXPORT_SYMBOL(__skb_checksum_complete);

2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726
static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
{
	net_warn_ratelimited(
		"%s: attempt to compute crc32c without libcrc32c.ko\n",
		__func__);
	return 0;
}

static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
				       int offset, int len)
{
	net_warn_ratelimited(
		"%s: attempt to compute crc32c without libcrc32c.ko\n",
		__func__);
	return 0;
}

static const struct skb_checksum_ops default_crc32c_ops = {
	.update  = warn_crc32c_csum_update,
	.combine = warn_crc32c_csum_combine,
};

const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
	&default_crc32c_ops;
EXPORT_SYMBOL(crc32c_csum_stub);

2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753
 /**
 *	skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
 *	@from: source buffer
 *
 *	Calculates the amount of linear headroom needed in the 'to' skb passed
 *	into skb_zerocopy().
 */
unsigned int
skb_zerocopy_headlen(const struct sk_buff *from)
{
	unsigned int hlen = 0;

	if (!from->head_frag ||
	    skb_headlen(from) < L1_CACHE_BYTES ||
	    skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
		hlen = skb_headlen(from);

	if (skb_has_frag_list(from))
		hlen = from->len;

	return hlen;
}
EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);

/**
 *	skb_zerocopy - Zero copy skb to skb
 *	@to: destination buffer
2754
 *	@from: source buffer
2755 2756 2757 2758 2759 2760 2761 2762
 *	@len: number of bytes to copy from source buffer
 *	@hlen: size of linear headroom in destination buffer
 *
 *	Copies up to `len` bytes from `from` to `to` by creating references
 *	to the frags in the source buffer.
 *
 *	The `hlen` as calculated by skb_zerocopy_headlen() specifies the
 *	headroom in the `to` buffer.
2763 2764 2765 2766 2767
 *
 *	Return value:
 *	0: everything is OK
 *	-ENOMEM: couldn't orphan frags of @from due to lack of memory
 *	-EFAULT: skb_copy_bits() found some problem with skb geometry
2768
 */
2769 2770
int
skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2771 2772 2773
{
	int i, j = 0;
	int plen = 0; /* length of skb->head fragment */
2774
	int ret;
2775 2776 2777 2778 2779 2780
	struct page *page;
	unsigned int offset;

	BUG_ON(!from->head_frag && !hlen);

	/* dont bother with small payloads */
2781 2782
	if (len <= skb_tailroom(to))
		return skb_copy_bits(from, 0, skb_put(to, len), len);
2783 2784

	if (hlen) {
2785 2786 2787
		ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
		if (unlikely(ret))
			return ret;
2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804
		len -= hlen;
	} else {
		plen = min_t(int, skb_headlen(from), len);
		if (plen) {
			page = virt_to_head_page(from->head);
			offset = from->data - (unsigned char *)page_address(page);
			__skb_fill_page_desc(to, 0, page, offset, plen);
			get_page(page);
			j = 1;
			len -= plen;
		}
	}

	to->truesize += len + plen;
	to->len += len + plen;
	to->data_len += len + plen;

2805 2806 2807 2808
	if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
		skb_tx_error(from);
		return -ENOMEM;
	}
W
Willem de Bruijn 已提交
2809
	skb_zerocopy_clone(to, from, GFP_ATOMIC);
2810

2811 2812 2813 2814 2815 2816 2817 2818 2819 2820
	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
		if (!len)
			break;
		skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
		skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
		len -= skb_shinfo(to)->frags[j].size;
		skb_frag_ref(to, j);
		j++;
	}
	skb_shinfo(to)->nr_frags = j;
2821 2822

	return 0;
2823 2824 2825
}
EXPORT_SYMBOL_GPL(skb_zerocopy);

L
Linus Torvalds 已提交
2826 2827
void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
{
2828
	__wsum csum;
L
Linus Torvalds 已提交
2829 2830
	long csstart;

2831
	if (skb->ip_summed == CHECKSUM_PARTIAL)
2832
		csstart = skb_checksum_start_offset(skb);
L
Linus Torvalds 已提交
2833 2834 2835
	else
		csstart = skb_headlen(skb);

2836
	BUG_ON(csstart > skb_headlen(skb));
L
Linus Torvalds 已提交
2837

2838
	skb_copy_from_linear_data(skb, to, csstart);
L
Linus Torvalds 已提交
2839 2840 2841 2842 2843 2844

	csum = 0;
	if (csstart != skb->len)
		csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
					      skb->len - csstart, 0);

2845
	if (skb->ip_summed == CHECKSUM_PARTIAL) {
A
Al Viro 已提交
2846
		long csstuff = csstart + skb->csum_offset;
L
Linus Torvalds 已提交
2847

2848
		*((__sum16 *)(to + csstuff)) = csum_fold(csum);
L
Linus Torvalds 已提交
2849 2850
	}
}
2851
EXPORT_SYMBOL(skb_copy_and_csum_dev);
L
Linus Torvalds 已提交
2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871

/**
 *	skb_dequeue - remove from the head of the queue
 *	@list: list to dequeue from
 *
 *	Remove the head of the list. The list lock is taken so the function
 *	may be used safely with other locking list functions. The head item is
 *	returned or %NULL if the list is empty.
 */

struct sk_buff *skb_dequeue(struct sk_buff_head *list)
{
	unsigned long flags;
	struct sk_buff *result;

	spin_lock_irqsave(&list->lock, flags);
	result = __skb_dequeue(list);
	spin_unlock_irqrestore(&list->lock, flags);
	return result;
}
2872
EXPORT_SYMBOL(skb_dequeue);
L
Linus Torvalds 已提交
2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891

/**
 *	skb_dequeue_tail - remove from the tail of the queue
 *	@list: list to dequeue from
 *
 *	Remove the tail of the list. The list lock is taken so the function
 *	may be used safely with other locking list functions. The tail item is
 *	returned or %NULL if the list is empty.
 */
struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
{
	unsigned long flags;
	struct sk_buff *result;

	spin_lock_irqsave(&list->lock, flags);
	result = __skb_dequeue_tail(list);
	spin_unlock_irqrestore(&list->lock, flags);
	return result;
}
2892
EXPORT_SYMBOL(skb_dequeue_tail);
L
Linus Torvalds 已提交
2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907

/**
 *	skb_queue_purge - empty a list
 *	@list: list to empty
 *
 *	Delete all buffers on an &sk_buff list. Each buffer is removed from
 *	the list and one reference dropped. This function takes the list
 *	lock and is atomic with respect to other list locking functions.
 */
void skb_queue_purge(struct sk_buff_head *list)
{
	struct sk_buff *skb;
	while ((skb = skb_dequeue(list)) != NULL)
		kfree_skb(skb);
}
2908
EXPORT_SYMBOL(skb_queue_purge);
L
Linus Torvalds 已提交
2909

2910 2911 2912
/**
 *	skb_rbtree_purge - empty a skb rbtree
 *	@root: root of the rbtree to empty
2913
 *	Return value: the sum of truesizes of all purged skbs.
2914 2915 2916 2917 2918 2919
 *
 *	Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
 *	the list and one reference dropped. This function does not take
 *	any lock. Synchronization should be handled by the caller (e.g., TCP
 *	out-of-order queue is protected by the socket lock).
 */
2920
unsigned int skb_rbtree_purge(struct rb_root *root)
2921
{
E
Eric Dumazet 已提交
2922
	struct rb_node *p = rb_first(root);
2923
	unsigned int sum = 0;
2924

E
Eric Dumazet 已提交
2925 2926
	while (p) {
		struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
2927

E
Eric Dumazet 已提交
2928 2929
		p = rb_next(p);
		rb_erase(&skb->rbnode, root);
2930
		sum += skb->truesize;
E
Eric Dumazet 已提交
2931 2932
		kfree_skb(skb);
	}
2933
	return sum;
2934 2935
}

L
Linus Torvalds 已提交
2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954
/**
 *	skb_queue_head - queue a buffer at the list head
 *	@list: list to use
 *	@newsk: buffer to queue
 *
 *	Queue a buffer at the start of the list. This function takes the
 *	list lock and can be used safely with other locking &sk_buff functions
 *	safely.
 *
 *	A buffer cannot be placed on two lists at the same time.
 */
void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
{
	unsigned long flags;

	spin_lock_irqsave(&list->lock, flags);
	__skb_queue_head(list, newsk);
	spin_unlock_irqrestore(&list->lock, flags);
}
2955
EXPORT_SYMBOL(skb_queue_head);
L
Linus Torvalds 已提交
2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975

/**
 *	skb_queue_tail - queue a buffer at the list tail
 *	@list: list to use
 *	@newsk: buffer to queue
 *
 *	Queue a buffer at the tail of the list. This function takes the
 *	list lock and can be used safely with other locking &sk_buff functions
 *	safely.
 *
 *	A buffer cannot be placed on two lists at the same time.
 */
void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
{
	unsigned long flags;

	spin_lock_irqsave(&list->lock, flags);
	__skb_queue_tail(list, newsk);
	spin_unlock_irqrestore(&list->lock, flags);
}
2976
EXPORT_SYMBOL(skb_queue_tail);
D
David S. Miller 已提交
2977

L
Linus Torvalds 已提交
2978 2979 2980
/**
 *	skb_unlink	-	remove a buffer from a list
 *	@skb: buffer to remove
D
David S. Miller 已提交
2981
 *	@list: list to use
L
Linus Torvalds 已提交
2982
 *
D
David S. Miller 已提交
2983 2984
 *	Remove a packet from a list. The list locks are taken and this
 *	function is atomic with respect to other list locked calls
L
Linus Torvalds 已提交
2985
 *
D
David S. Miller 已提交
2986
 *	You must know what list the SKB is on.
L
Linus Torvalds 已提交
2987
 */
D
David S. Miller 已提交
2988
void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
L
Linus Torvalds 已提交
2989
{
D
David S. Miller 已提交
2990
	unsigned long flags;
L
Linus Torvalds 已提交
2991

D
David S. Miller 已提交
2992 2993 2994
	spin_lock_irqsave(&list->lock, flags);
	__skb_unlink(skb, list);
	spin_unlock_irqrestore(&list->lock, flags);
L
Linus Torvalds 已提交
2995
}
2996
EXPORT_SYMBOL(skb_unlink);
L
Linus Torvalds 已提交
2997 2998 2999 3000 3001

/**
 *	skb_append	-	append a buffer
 *	@old: buffer to insert after
 *	@newsk: buffer to insert
D
David S. Miller 已提交
3002
 *	@list: list to use
L
Linus Torvalds 已提交
3003 3004 3005 3006 3007
 *
 *	Place a packet after a given packet in a list. The list locks are taken
 *	and this function is atomic with respect to other list locked calls.
 *	A buffer cannot be placed on two lists at the same time.
 */
D
David S. Miller 已提交
3008
void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
L
Linus Torvalds 已提交
3009 3010 3011
{
	unsigned long flags;

D
David S. Miller 已提交
3012
	spin_lock_irqsave(&list->lock, flags);
3013
	__skb_queue_after(list, old, newsk);
D
David S. Miller 已提交
3014
	spin_unlock_irqrestore(&list->lock, flags);
L
Linus Torvalds 已提交
3015
}
3016
EXPORT_SYMBOL(skb_append);
L
Linus Torvalds 已提交
3017 3018 3019 3020 3021 3022 3023

static inline void skb_split_inside_header(struct sk_buff *skb,
					   struct sk_buff* skb1,
					   const u32 len, const int pos)
{
	int i;

3024 3025
	skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
					 pos - len);
L
Linus Torvalds 已提交
3026 3027 3028 3029 3030 3031 3032 3033 3034 3035
	/* And move data appendix as is. */
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
		skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];

	skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
	skb_shinfo(skb)->nr_frags  = 0;
	skb1->data_len		   = skb->data_len;
	skb1->len		   += skb1->data_len;
	skb->data_len		   = 0;
	skb->len		   = len;
3036
	skb_set_tail_pointer(skb, len);
L
Linus Torvalds 已提交
3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051
}

static inline void skb_split_no_header(struct sk_buff *skb,
				       struct sk_buff* skb1,
				       const u32 len, int pos)
{
	int i, k = 0;
	const int nfrags = skb_shinfo(skb)->nr_frags;

	skb_shinfo(skb)->nr_frags = 0;
	skb1->len		  = skb1->data_len = skb->len - len;
	skb->len		  = len;
	skb->data_len		  = len - pos;

	for (i = 0; i < nfrags; i++) {
E
Eric Dumazet 已提交
3052
		int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
L
Linus Torvalds 已提交
3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065

		if (pos + size > len) {
			skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];

			if (pos < len) {
				/* Split frag.
				 * We have two variants in this case:
				 * 1. Move all the frag to the second
				 *    part, if it is possible. F.e.
				 *    this approach is mandatory for TUX,
				 *    where splitting is expensive.
				 * 2. Split is accurately. We make this.
				 */
3066
				skb_frag_ref(skb, i);
L
Linus Torvalds 已提交
3067
				skb_shinfo(skb1)->frags[0].page_offset += len - pos;
E
Eric Dumazet 已提交
3068 3069
				skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
				skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
L
Linus Torvalds 已提交
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089
				skb_shinfo(skb)->nr_frags++;
			}
			k++;
		} else
			skb_shinfo(skb)->nr_frags++;
		pos += size;
	}
	skb_shinfo(skb1)->nr_frags = k;
}

/**
 * skb_split - Split fragmented skb to two parts at length len.
 * @skb: the buffer to split
 * @skb1: the buffer to receive the second part
 * @len: new length for skb
 */
void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
{
	int pos = skb_headlen(skb);

3090 3091
	skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
				      SKBTX_SHARED_FRAG;
W
Willem de Bruijn 已提交
3092
	skb_zerocopy_clone(skb1, skb, 0);
L
Linus Torvalds 已提交
3093 3094 3095 3096 3097
	if (len < pos)	/* Split line is inside header. */
		skb_split_inside_header(skb, skb1, len, pos);
	else		/* Second chunk has no header, nothing to copy. */
		skb_split_no_header(skb, skb1, len, pos);
}
3098
EXPORT_SYMBOL(skb_split);
L
Linus Torvalds 已提交
3099

3100 3101 3102 3103
/* Shifting from/to a cloned skb is a no-go.
 *
 * Caller cannot keep skb_shinfo related pointers past calling here!
 */
3104 3105
static int skb_prepare_for_shift(struct sk_buff *skb)
{
3106
	return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3107 3108 3109 3110 3111 3112 3113 3114 3115
}

/**
 * skb_shift - Shifts paged data partially from skb to another
 * @tgt: buffer into which tail data gets added
 * @skb: buffer from which the paged data comes from
 * @shiftlen: shift up to this many bytes
 *
 * Attempts to shift up to shiftlen worth of bytes, which may be less than
F
Feng King 已提交
3116
 * the length of the skb, from skb to tgt. Returns number bytes shifted.
3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132
 * It's up to caller to free skb if everything was shifted.
 *
 * If @tgt runs out of frags, the whole operation is aborted.
 *
 * Skb cannot include anything else but paged data while tgt is allowed
 * to have non-paged data as well.
 *
 * TODO: full sized shift could be optimized but that would need
 * specialized skb free'er to handle frags without up-to-date nr_frags.
 */
int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
{
	int from, to, merge, todo;
	struct skb_frag_struct *fragfrom, *fragto;

	BUG_ON(shiftlen > skb->len);
3133 3134 3135

	if (skb_headlen(skb))
		return 0;
W
Willem de Bruijn 已提交
3136 3137
	if (skb_zcopy(tgt) || skb_zcopy(skb))
		return 0;
3138 3139 3140 3141 3142 3143 3144 3145 3146 3147

	todo = shiftlen;
	from = 0;
	to = skb_shinfo(tgt)->nr_frags;
	fragfrom = &skb_shinfo(skb)->frags[from];

	/* Actual merge is delayed until the point when we know we can
	 * commit all, so that we don't have to undo partial changes
	 */
	if (!to ||
3148 3149
	    !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
			      fragfrom->page_offset)) {
3150 3151 3152 3153
		merge = -1;
	} else {
		merge = to - 1;

E
Eric Dumazet 已提交
3154
		todo -= skb_frag_size(fragfrom);
3155 3156 3157 3158 3159
		if (todo < 0) {
			if (skb_prepare_for_shift(skb) ||
			    skb_prepare_for_shift(tgt))
				return 0;

3160 3161
			/* All previous frag pointers might be stale! */
			fragfrom = &skb_shinfo(skb)->frags[from];
3162 3163
			fragto = &skb_shinfo(tgt)->frags[merge];

E
Eric Dumazet 已提交
3164 3165
			skb_frag_size_add(fragto, shiftlen);
			skb_frag_size_sub(fragfrom, shiftlen);
3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188
			fragfrom->page_offset += shiftlen;

			goto onlymerged;
		}

		from++;
	}

	/* Skip full, not-fitting skb to avoid expensive operations */
	if ((shiftlen == skb->len) &&
	    (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
		return 0;

	if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
		return 0;

	while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
		if (to == MAX_SKB_FRAGS)
			return 0;

		fragfrom = &skb_shinfo(skb)->frags[from];
		fragto = &skb_shinfo(tgt)->frags[to];

E
Eric Dumazet 已提交
3189
		if (todo >= skb_frag_size(fragfrom)) {
3190
			*fragto = *fragfrom;
E
Eric Dumazet 已提交
3191
			todo -= skb_frag_size(fragfrom);
3192 3193 3194 3195
			from++;
			to++;

		} else {
3196
			__skb_frag_ref(fragfrom);
3197 3198
			fragto->page = fragfrom->page;
			fragto->page_offset = fragfrom->page_offset;
E
Eric Dumazet 已提交
3199
			skb_frag_size_set(fragto, todo);
3200 3201

			fragfrom->page_offset += todo;
E
Eric Dumazet 已提交
3202
			skb_frag_size_sub(fragfrom, todo);
3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216
			todo = 0;

			to++;
			break;
		}
	}

	/* Ready to "commit" this state change to tgt */
	skb_shinfo(tgt)->nr_frags = to;

	if (merge >= 0) {
		fragfrom = &skb_shinfo(skb)->frags[0];
		fragto = &skb_shinfo(tgt)->frags[merge];

E
Eric Dumazet 已提交
3217
		skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3218
		__skb_frag_unref(fragfrom);
3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246
	}

	/* Reposition in the original skb */
	to = 0;
	while (from < skb_shinfo(skb)->nr_frags)
		skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
	skb_shinfo(skb)->nr_frags = to;

	BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);

onlymerged:
	/* Most likely the tgt won't ever need its checksum anymore, skb on
	 * the other hand might need it if it needs to be resent
	 */
	tgt->ip_summed = CHECKSUM_PARTIAL;
	skb->ip_summed = CHECKSUM_PARTIAL;

	/* Yak, is it really working this way? Some helper please? */
	skb->len -= shiftlen;
	skb->data_len -= shiftlen;
	skb->truesize -= shiftlen;
	tgt->len += shiftlen;
	tgt->data_len += shiftlen;
	tgt->truesize += shiftlen;

	return shiftlen;
}

3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265
/**
 * skb_prepare_seq_read - Prepare a sequential read of skb data
 * @skb: the buffer to read
 * @from: lower offset of data to be read
 * @to: upper offset of data to be read
 * @st: state variable
 *
 * Initializes the specified state variable. Must be called before
 * invoking skb_seq_read() for the first time.
 */
void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
			  unsigned int to, struct skb_seq_state *st)
{
	st->lower_offset = from;
	st->upper_offset = to;
	st->root_skb = st->cur_skb = skb;
	st->frag_idx = st->stepped_offset = 0;
	st->frag_data = NULL;
}
3266
EXPORT_SYMBOL(skb_prepare_seq_read);
3267 3268 3269 3270 3271 3272 3273

/**
 * skb_seq_read - Sequentially read skb data
 * @consumed: number of bytes consumed by the caller so far
 * @data: destination pointer for data to be returned
 * @st: state variable
 *
M
Mathias Krause 已提交
3274
 * Reads a block of skb data at @consumed relative to the
3275
 * lower offset specified to skb_prepare_seq_read(). Assigns
M
Mathias Krause 已提交
3276
 * the head of the data block to @data and returns the length
3277 3278 3279 3280
 * of the block or 0 if the end of the skb data or the upper
 * offset has been reached.
 *
 * The caller is not required to consume all of the data
M
Mathias Krause 已提交
3281
 * returned, i.e. @consumed is typically set to the number
3282 3283 3284
 * of bytes already consumed and the next call to
 * skb_seq_read() will return the remaining part of the block.
 *
L
Lucas De Marchi 已提交
3285
 * Note 1: The size of each block of data returned can be arbitrary,
3286
 *       this limitation is the cost for zerocopy sequential
3287 3288
 *       reads of potentially non linear data.
 *
3289
 * Note 2: Fragment lists within fragments are not implemented
3290 3291 3292 3293 3294 3295 3296 3297 3298
 *       at the moment, state->root_skb could be replaced with
 *       a stack for this purpose.
 */
unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
			  struct skb_seq_state *st)
{
	unsigned int block_limit, abs_offset = consumed + st->lower_offset;
	skb_frag_t *frag;

3299 3300 3301 3302 3303
	if (unlikely(abs_offset >= st->upper_offset)) {
		if (st->frag_data) {
			kunmap_atomic(st->frag_data);
			st->frag_data = NULL;
		}
3304
		return 0;
3305
	}
3306 3307

next_skb:
3308
	block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
3309

3310
	if (abs_offset < block_limit && !st->frag_data) {
3311
		*data = st->cur_skb->data + (abs_offset - st->stepped_offset);
3312 3313 3314 3315 3316 3317 3318 3319
		return block_limit - abs_offset;
	}

	if (st->frag_idx == 0 && !st->frag_data)
		st->stepped_offset += skb_headlen(st->cur_skb);

	while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
		frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
E
Eric Dumazet 已提交
3320
		block_limit = skb_frag_size(frag) + st->stepped_offset;
3321 3322 3323

		if (abs_offset < block_limit) {
			if (!st->frag_data)
E
Eric Dumazet 已提交
3324
				st->frag_data = kmap_atomic(skb_frag_page(frag));
3325 3326 3327 3328 3329 3330 3331 3332

			*data = (u8 *) st->frag_data + frag->page_offset +
				(abs_offset - st->stepped_offset);

			return block_limit - abs_offset;
		}

		if (st->frag_data) {
E
Eric Dumazet 已提交
3333
			kunmap_atomic(st->frag_data);
3334 3335 3336 3337
			st->frag_data = NULL;
		}

		st->frag_idx++;
E
Eric Dumazet 已提交
3338
		st->stepped_offset += skb_frag_size(frag);
3339 3340
	}

3341
	if (st->frag_data) {
E
Eric Dumazet 已提交
3342
		kunmap_atomic(st->frag_data);
3343 3344 3345
		st->frag_data = NULL;
	}

3346
	if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
S
Shyam Iyer 已提交
3347
		st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
3348 3349
		st->frag_idx = 0;
		goto next_skb;
S
Shyam Iyer 已提交
3350 3351
	} else if (st->cur_skb->next) {
		st->cur_skb = st->cur_skb->next;
3352
		st->frag_idx = 0;
3353 3354 3355 3356 3357
		goto next_skb;
	}

	return 0;
}
3358
EXPORT_SYMBOL(skb_seq_read);
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369

/**
 * skb_abort_seq_read - Abort a sequential read of skb data
 * @st: state variable
 *
 * Must be called if skb_seq_read() was not called until it
 * returned 0.
 */
void skb_abort_seq_read(struct skb_seq_state *st)
{
	if (st->frag_data)
E
Eric Dumazet 已提交
3370
		kunmap_atomic(st->frag_data);
3371
}
3372
EXPORT_SYMBOL(skb_abort_seq_read);
3373

3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400
#define TS_SKB_CB(state)	((struct skb_seq_state *) &((state)->cb))

static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
					  struct ts_config *conf,
					  struct ts_state *state)
{
	return skb_seq_read(offset, text, TS_SKB_CB(state));
}

static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
{
	skb_abort_seq_read(TS_SKB_CB(state));
}

/**
 * skb_find_text - Find a text pattern in skb data
 * @skb: the buffer to look in
 * @from: search offset
 * @to: search limit
 * @config: textsearch configuration
 *
 * Finds a pattern in the skb data according to the specified
 * textsearch configuration. Use textsearch_next() to retrieve
 * subsequent occurrences of the pattern. Returns the offset
 * to the first occurrence or UINT_MAX if no match was found.
 */
unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
3401
			   unsigned int to, struct ts_config *config)
3402
{
3403
	struct ts_state state;
3404 3405
	unsigned int ret;

3406 3407 3408
	config->get_next_block = skb_ts_get_next_block;
	config->finish = skb_ts_finish;

3409
	skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
3410

3411
	ret = textsearch_find(config, &state);
3412
	return (ret <= to - from ? ret : UINT_MAX);
3413
}
3414
EXPORT_SYMBOL(skb_find_text);
3415

3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433
int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
			 int offset, size_t size)
{
	int i = skb_shinfo(skb)->nr_frags;

	if (skb_can_coalesce(skb, i, page, offset)) {
		skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
	} else if (i < MAX_SKB_FRAGS) {
		get_page(page);
		skb_fill_page_desc(skb, i, page, offset, size);
	} else {
		return -EMSGSIZE;
	}

	return 0;
}
EXPORT_SYMBOL_GPL(skb_append_pagefrags);

3434 3435 3436 3437 3438 3439
/**
 *	skb_pull_rcsum - pull skb and update receive checksum
 *	@skb: buffer to update
 *	@len: length of data pulled
 *
 *	This function performs an skb_pull on the packet and updates
3440
 *	the CHECKSUM_COMPLETE checksum.  It should be used on
3441 3442 3443
 *	receive path processing instead of skb_pull unless you know
 *	that the checksum difference is zero (e.g., a valid IP header)
 *	or you are setting ip_summed to CHECKSUM_NONE.
3444
 */
3445
void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
3446
{
3447 3448
	unsigned char *data = skb->data;

3449
	BUG_ON(len > skb->len);
3450 3451 3452
	__skb_pull(skb, len);
	skb_postpull_rcsum(skb, data, len);
	return skb->data;
3453
}
3454 3455
EXPORT_SYMBOL_GPL(skb_pull_rcsum);

3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468
static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
{
	skb_frag_t head_frag;
	struct page *page;

	page = virt_to_head_page(frag_skb->head);
	head_frag.page.p = page;
	head_frag.page_offset = frag_skb->data -
		(unsigned char *)page_address(page);
	head_frag.size = skb_headlen(frag_skb);
	return head_frag;
}

H
Herbert Xu 已提交
3469 3470
/**
 *	skb_segment - Perform protocol segmentation on skb.
3471
 *	@head_skb: buffer to segment
3472
 *	@features: features for the output path (see dev->features)
H
Herbert Xu 已提交
3473 3474
 *
 *	This function performs segmentation on the given skb.  It returns
3475 3476
 *	a pointer to the first in a list of new skbs for the segments.
 *	In case of error it returns ERR_PTR(err).
H
Herbert Xu 已提交
3477
 */
3478 3479
struct sk_buff *skb_segment(struct sk_buff *head_skb,
			    netdev_features_t features)
H
Herbert Xu 已提交
3480 3481 3482
{
	struct sk_buff *segs = NULL;
	struct sk_buff *tail = NULL;
3483
	struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
3484 3485 3486
	skb_frag_t *frag = skb_shinfo(head_skb)->frags;
	unsigned int mss = skb_shinfo(head_skb)->gso_size;
	unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
3487
	struct sk_buff *frag_skb = head_skb;
H
Herbert Xu 已提交
3488
	unsigned int offset = doffset;
3489
	unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
3490
	unsigned int partial_segs = 0;
H
Herbert Xu 已提交
3491
	unsigned int headroom;
3492
	unsigned int len = head_skb->len;
3493
	__be16 proto;
3494
	bool csum, sg;
3495
	int nfrags = skb_shinfo(head_skb)->nr_frags;
H
Herbert Xu 已提交
3496 3497 3498
	int err = -ENOMEM;
	int i = 0;
	int pos;
3499
	int dummy;
H
Herbert Xu 已提交
3500

3501
	__skb_push(head_skb, doffset);
3502
	proto = skb_network_protocol(head_skb, &dummy);
3503 3504 3505
	if (unlikely(!proto))
		return ERR_PTR(-EINVAL);

3506
	sg = !!(features & NETIF_F_SG);
3507
	csum = !!can_checksum_protocol(features, proto);
3508

3509 3510 3511
	if (sg && csum && (mss != GSO_BY_FRAGS))  {
		if (!(features & NETIF_F_GSO_PARTIAL)) {
			struct sk_buff *iter;
3512
			unsigned int frag_len;
3513 3514 3515 3516 3517

			if (!list_skb ||
			    !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
				goto normal;

3518 3519 3520 3521 3522 3523 3524
			/* If we get here then all the required
			 * GSO features except frag_list are supported.
			 * Try to split the SKB to multiple GSO SKBs
			 * with no frag_list.
			 * Currently we can do that only when the buffers don't
			 * have a linear part and all the buffers except
			 * the last are of the same length.
3525
			 */
3526
			frag_len = list_skb->len;
3527
			skb_walk_frags(head_skb, iter) {
3528 3529
				if (frag_len != iter->len && iter->next)
					goto normal;
3530
				if (skb_headlen(iter) && !iter->head_frag)
3531 3532 3533 3534
					goto normal;

				len -= iter->len;
			}
3535 3536 3537

			if (len != frag_len)
				goto normal;
3538 3539 3540 3541 3542 3543
		}

		/* GSO partial only requires that we trim off any excess that
		 * doesn't fit into an MSS sized block, so take care of that
		 * now.
		 */
3544
		partial_segs = len / mss;
3545 3546 3547 3548
		if (partial_segs > 1)
			mss *= partial_segs;
		else
			partial_segs = 0;
3549 3550
	}

3551
normal:
3552 3553
	headroom = skb_headroom(head_skb);
	pos = skb_headlen(head_skb);
H
Herbert Xu 已提交
3554 3555 3556

	do {
		struct sk_buff *nskb;
3557
		skb_frag_t *nskb_frag;
3558
		int hsize;
H
Herbert Xu 已提交
3559 3560
		int size;

3561 3562 3563 3564 3565 3566 3567
		if (unlikely(mss == GSO_BY_FRAGS)) {
			len = list_skb->len;
		} else {
			len = head_skb->len - offset;
			if (len > mss)
				len = mss;
		}
H
Herbert Xu 已提交
3568

3569
		hsize = skb_headlen(head_skb) - offset;
H
Herbert Xu 已提交
3570 3571
		if (hsize < 0)
			hsize = 0;
3572 3573
		if (hsize > len || !sg)
			hsize = len;
H
Herbert Xu 已提交
3574

3575 3576 3577
		if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
		    (skb_headlen(list_skb) == len || sg)) {
			BUG_ON(skb_headlen(list_skb) > len);
3578 3579

			i = 0;
3580 3581
			nfrags = skb_shinfo(list_skb)->nr_frags;
			frag = skb_shinfo(list_skb)->frags;
3582
			frag_skb = list_skb;
3583
			pos += skb_headlen(list_skb);
3584 3585 3586 3587

			while (pos < offset + len) {
				BUG_ON(i >= nfrags);

3588
				size = skb_frag_size(frag);
3589 3590 3591 3592 3593
				if (pos + size > offset + len)
					break;

				i++;
				pos += size;
3594
				frag++;
3595
			}
3596

3597 3598
			nskb = skb_clone(list_skb, GFP_ATOMIC);
			list_skb = list_skb->next;
3599 3600 3601 3602

			if (unlikely(!nskb))
				goto err;

3603 3604 3605 3606 3607
			if (unlikely(pskb_trim(nskb, len))) {
				kfree_skb(nskb);
				goto err;
			}

3608
			hsize = skb_end_offset(nskb);
3609 3610 3611 3612 3613
			if (skb_cow_head(nskb, doffset + headroom)) {
				kfree_skb(nskb);
				goto err;
			}

3614
			nskb->truesize += skb_end_offset(nskb) - hsize;
3615 3616 3617
			skb_release_head_state(nskb);
			__skb_push(nskb, doffset);
		} else {
3618
			nskb = __alloc_skb(hsize + doffset + headroom,
3619
					   GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
3620
					   NUMA_NO_NODE);
3621 3622 3623 3624 3625 3626 3627

			if (unlikely(!nskb))
				goto err;

			skb_reserve(nskb, headroom);
			__skb_put(nskb, doffset);
		}
H
Herbert Xu 已提交
3628 3629 3630 3631 3632 3633 3634

		if (segs)
			tail->next = nskb;
		else
			segs = nskb;
		tail = nskb;

3635
		__copy_skb_header(nskb, head_skb);
H
Herbert Xu 已提交
3636

E
Eric Dumazet 已提交
3637
		skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
3638
		skb_reset_mac_len(nskb);
3639

3640
		skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
3641 3642
						 nskb->data - tnl_hlen,
						 doffset + tnl_hlen);
3643

3644
		if (nskb->len == len + doffset)
3645
			goto perform_csum_check;
3646

3647 3648 3649
		if (!sg) {
			if (!nskb->remcsum_offload)
				nskb->ip_summed = CHECKSUM_NONE;
3650 3651 3652 3653
			SKB_GSO_CB(nskb)->csum =
				skb_copy_and_csum_bits(head_skb, offset,
						       skb_put(nskb, len),
						       len, 0);
3654
			SKB_GSO_CB(nskb)->csum_start =
3655
				skb_headroom(nskb) + doffset;
H
Herbert Xu 已提交
3656 3657 3658
			continue;
		}

3659
		nskb_frag = skb_shinfo(nskb)->frags;
H
Herbert Xu 已提交
3660

3661
		skb_copy_from_linear_data_offset(head_skb, offset,
3662
						 skb_put(nskb, hsize), hsize);
H
Herbert Xu 已提交
3663

3664 3665
		skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
					      SKBTX_SHARED_FRAG;
3666

3667 3668 3669 3670
		if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
		    skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
			goto err;

3671 3672 3673
		while (pos < offset + len) {
			if (i >= nfrags) {
				i = 0;
3674 3675
				nfrags = skb_shinfo(list_skb)->nr_frags;
				frag = skb_shinfo(list_skb)->frags;
3676
				frag_skb = list_skb;
3677 3678 3679 3680
				if (!skb_headlen(list_skb)) {
					BUG_ON(!nfrags);
				} else {
					BUG_ON(!list_skb->head_frag);
3681

3682 3683 3684 3685
					/* to make room for head_frag. */
					i--;
					frag--;
				}
3686 3687 3688 3689 3690
				if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
				    skb_zerocopy_clone(nskb, frag_skb,
						       GFP_ATOMIC))
					goto err;

3691
				list_skb = list_skb->next;
3692 3693 3694 3695 3696 3697 3698
			}

			if (unlikely(skb_shinfo(nskb)->nr_frags >=
				     MAX_SKB_FRAGS)) {
				net_warn_ratelimited(
					"skb_segment: too many frags: %u %u\n",
					pos, mss);
3699
				err = -EINVAL;
3700 3701 3702
				goto err;
			}

3703
			*nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
3704 3705
			__skb_frag_ref(nskb_frag);
			size = skb_frag_size(nskb_frag);
H
Herbert Xu 已提交
3706 3707

			if (pos < offset) {
3708 3709
				nskb_frag->page_offset += offset - pos;
				skb_frag_size_sub(nskb_frag, offset - pos);
H
Herbert Xu 已提交
3710 3711
			}

3712
			skb_shinfo(nskb)->nr_frags++;
H
Herbert Xu 已提交
3713 3714 3715

			if (pos + size <= offset + len) {
				i++;
3716
				frag++;
H
Herbert Xu 已提交
3717 3718
				pos += size;
			} else {
3719
				skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
3720
				goto skip_fraglist;
H
Herbert Xu 已提交
3721 3722
			}

3723
			nskb_frag++;
H
Herbert Xu 已提交
3724 3725
		}

3726
skip_fraglist:
H
Herbert Xu 已提交
3727 3728 3729
		nskb->data_len = len - hsize;
		nskb->len += nskb->data_len;
		nskb->truesize += nskb->data_len;
3730

3731
perform_csum_check:
3732
		if (!csum) {
3733 3734 3735 3736
			if (skb_has_shared_frag(nskb) &&
			    __skb_linearize(nskb))
				goto err;

3737 3738
			if (!nskb->remcsum_offload)
				nskb->ip_summed = CHECKSUM_NONE;
3739 3740 3741
			SKB_GSO_CB(nskb)->csum =
				skb_checksum(nskb, doffset,
					     nskb->len - doffset, 0);
3742
			SKB_GSO_CB(nskb)->csum_start =
3743
				skb_headroom(nskb) + doffset;
3744
		}
3745
	} while ((offset += len) < head_skb->len);
H
Herbert Xu 已提交
3746

3747 3748 3749 3750 3751
	/* Some callers want to get the end of the list.
	 * Put it in segs->prev to avoid walking the list.
	 * (see validate_xmit_skb_list() for example)
	 */
	segs->prev = tail;
3752

3753
	if (partial_segs) {
3754
		struct sk_buff *iter;
3755
		int type = skb_shinfo(head_skb)->gso_type;
3756
		unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
3757 3758

		/* Update type to add partial and then remove dodgy if set */
3759
		type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
3760 3761 3762 3763 3764
		type &= ~SKB_GSO_DODGY;

		/* Update GSO info and prepare to start updating headers on
		 * our way back down the stack of protocols.
		 */
3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775
		for (iter = segs; iter; iter = iter->next) {
			skb_shinfo(iter)->gso_size = gso_size;
			skb_shinfo(iter)->gso_segs = partial_segs;
			skb_shinfo(iter)->gso_type = type;
			SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
		}

		if (tail->len - doffset <= gso_size)
			skb_shinfo(tail)->gso_size = 0;
		else if (tail != segs)
			skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
3776 3777
	}

3778 3779 3780 3781 3782 3783 3784 3785 3786
	/* Following permits correct backpressure, for protocols
	 * using skb_set_owner_w().
	 * Idea is to tranfert ownership from head_skb to last segment.
	 */
	if (head_skb->destructor == sock_wfree) {
		swap(tail->truesize, head_skb->truesize);
		swap(tail->destructor, head_skb->destructor);
		swap(tail->sk, head_skb->sk);
	}
H
Herbert Xu 已提交
3787 3788 3789
	return segs;

err:
E
Eric Dumazet 已提交
3790
	kfree_skb_list(segs);
H
Herbert Xu 已提交
3791 3792 3793 3794
	return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(skb_segment);

3795
int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
H
Herbert Xu 已提交
3796
{
3797
	struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
3798 3799
	unsigned int offset = skb_gro_offset(skb);
	unsigned int headlen = skb_headlen(skb);
3800
	unsigned int len = skb_gro_len(skb);
3801
	unsigned int delta_truesize;
3802
	struct sk_buff *lp;
H
Herbert Xu 已提交
3803

3804
	if (unlikely(p->len + len >= 65536))
H
Herbert Xu 已提交
3805 3806
		return -E2BIG;

3807
	lp = NAPI_GRO_CB(p)->last;
3808 3809 3810
	pinfo = skb_shinfo(lp);

	if (headlen <= offset) {
3811
		skb_frag_t *frag;
3812
		skb_frag_t *frag2;
3813 3814
		int i = skbinfo->nr_frags;
		int nr_frags = pinfo->nr_frags + i;
3815 3816

		if (nr_frags > MAX_SKB_FRAGS)
3817
			goto merge;
3818

3819
		offset -= headlen;
3820 3821
		pinfo->nr_frags = nr_frags;
		skbinfo->nr_frags = 0;
3822

3823 3824
		frag = pinfo->frags + nr_frags;
		frag2 = skbinfo->frags + i;
3825 3826 3827
		do {
			*--frag = *--frag2;
		} while (--i);
H
Herbert Xu 已提交
3828

3829
		frag->page_offset += offset;
E
Eric Dumazet 已提交
3830
		skb_frag_size_sub(frag, offset);
3831

3832
		/* all fragments truesize : remove (head size + sk_buff) */
3833 3834
		delta_truesize = skb->truesize -
				 SKB_TRUESIZE(skb_end_offset(skb));
3835

3836 3837 3838 3839
		skb->truesize -= skb->data_len;
		skb->len -= skb->data_len;
		skb->data_len = 0;

3840
		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
H
Herbert Xu 已提交
3841
		goto done;
3842 3843 3844 3845 3846 3847 3848 3849
	} else if (skb->head_frag) {
		int nr_frags = pinfo->nr_frags;
		skb_frag_t *frag = pinfo->frags + nr_frags;
		struct page *page = virt_to_head_page(skb->head);
		unsigned int first_size = headlen - offset;
		unsigned int first_offset;

		if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
3850
			goto merge;
3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864

		first_offset = skb->data -
			       (unsigned char *)page_address(page) +
			       offset;

		pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;

		frag->page.p	  = page;
		frag->page_offset = first_offset;
		skb_frag_size_set(frag, first_size);

		memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
		/* We dont need to clear skbinfo->nr_frags here */

3865
		delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3866 3867
		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
		goto done;
3868
	}
H
Herbert Xu 已提交
3869 3870

merge:
3871
	delta_truesize = skb->truesize;
3872
	if (offset > headlen) {
3873 3874 3875
		unsigned int eat = offset - headlen;

		skbinfo->frags[0].page_offset += eat;
E
Eric Dumazet 已提交
3876
		skb_frag_size_sub(&skbinfo->frags[0], eat);
3877 3878
		skb->data_len -= eat;
		skb->len -= eat;
3879
		offset = headlen;
3880 3881
	}

3882
	__skb_pull(skb, offset);
3883

3884
	if (NAPI_GRO_CB(p)->last == p)
3885 3886 3887
		skb_shinfo(p)->frag_list = skb;
	else
		NAPI_GRO_CB(p)->last->next = skb;
3888
	NAPI_GRO_CB(p)->last = skb;
3889
	__skb_header_release(skb);
3890
	lp = p;
H
Herbert Xu 已提交
3891

H
Herbert Xu 已提交
3892 3893
done:
	NAPI_GRO_CB(p)->count++;
H
Herbert Xu 已提交
3894
	p->data_len += len;
3895
	p->truesize += delta_truesize;
H
Herbert Xu 已提交
3896
	p->len += len;
3897 3898 3899 3900 3901
	if (lp != p) {
		lp->data_len += len;
		lp->truesize += delta_truesize;
		lp->len += len;
	}
H
Herbert Xu 已提交
3902 3903 3904
	NAPI_GRO_CB(skb)->same_flow = 1;
	return 0;
}
3905
EXPORT_SYMBOL_GPL(skb_gro_receive);
H
Herbert Xu 已提交
3906

3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940
#ifdef CONFIG_SKB_EXTENSIONS
#define SKB_EXT_ALIGN_VALUE	8
#define SKB_EXT_CHUNKSIZEOF(x)	(ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)

static const u8 skb_ext_type_len[] = {
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
	[SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
#endif
};

static __always_inline unsigned int skb_ext_total_length(void)
{
	return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
		skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
#endif
		0;
}

static void skb_extensions_init(void)
{
	BUILD_BUG_ON(SKB_EXT_NUM >= 8);
	BUILD_BUG_ON(skb_ext_total_length() > 255);

	skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
					     SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
					     0,
					     SLAB_HWCACHE_ALIGN|SLAB_PANIC,
					     NULL);
}
#else
static void skb_extensions_init(void) {}
#endif

L
Linus Torvalds 已提交
3941 3942
void __init skb_init(void)
{
3943
	skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
L
Linus Torvalds 已提交
3944 3945
					      sizeof(struct sk_buff),
					      0,
A
Alexey Dobriyan 已提交
3946
					      SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3947 3948
					      offsetof(struct sk_buff, cb),
					      sizeof_field(struct sk_buff, cb),
3949
					      NULL);
3950
	skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3951
						sizeof(struct sk_buff_fclones),
3952
						0,
A
Alexey Dobriyan 已提交
3953
						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3954
						NULL);
3955
	skb_extensions_init();
L
Linus Torvalds 已提交
3956 3957
}

3958
static int
3959 3960
__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
	       unsigned int recursion_level)
3961
{
3962 3963
	int start = skb_headlen(skb);
	int i, copy = start - offset;
3964
	struct sk_buff *frag_iter;
3965 3966
	int elt = 0;

3967 3968 3969
	if (unlikely(recursion_level >= 24))
		return -EMSGSIZE;

3970 3971 3972
	if (copy > 0) {
		if (copy > len)
			copy = len;
3973
		sg_set_buf(sg, skb->data + offset, copy);
3974 3975 3976 3977 3978 3979 3980
		elt++;
		if ((len -= copy) == 0)
			return elt;
		offset += copy;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3981
		int end;
3982

3983
		WARN_ON(start > offset + len);
3984

E
Eric Dumazet 已提交
3985
		end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3986 3987
		if ((copy = end - offset) > 0) {
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3988 3989
			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
				return -EMSGSIZE;
3990 3991 3992

			if (copy > len)
				copy = len;
3993
			sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3994
					frag->page_offset+offset-start);
3995 3996 3997 3998 3999
			elt++;
			if (!(len -= copy))
				return elt;
			offset += copy;
		}
4000
		start = end;
4001 4002
	}

4003
	skb_walk_frags(skb, frag_iter) {
4004
		int end, ret;
4005

4006
		WARN_ON(start > offset + len);
4007

4008 4009
		end = start + frag_iter->len;
		if ((copy = end - offset) > 0) {
4010 4011 4012
			if (unlikely(elt && sg_is_last(&sg[elt - 1])))
				return -EMSGSIZE;

4013 4014
			if (copy > len)
				copy = len;
4015 4016 4017 4018 4019
			ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
					      copy, recursion_level + 1);
			if (unlikely(ret < 0))
				return ret;
			elt += ret;
4020 4021 4022
			if ((len -= copy) == 0)
				return elt;
			offset += copy;
4023
		}
4024
		start = end;
4025 4026 4027 4028 4029
	}
	BUG_ON(len);
	return elt;
}

4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054
/**
 *	skb_to_sgvec - Fill a scatter-gather list from a socket buffer
 *	@skb: Socket buffer containing the buffers to be mapped
 *	@sg: The scatter-gather list to map into
 *	@offset: The offset into the buffer's contents to start mapping
 *	@len: Length of buffer space to be mapped
 *
 *	Fill the specified scatter-gather list with mappings/pointers into a
 *	region of the buffer space attached to a socket buffer. Returns either
 *	the number of scatterlist items used, or -EMSGSIZE if the contents
 *	could not fit.
 */
int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
{
	int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);

	if (nsg <= 0)
		return nsg;

	sg_mark_end(&sg[nsg - 1]);

	return nsg;
}
EXPORT_SYMBOL_GPL(skb_to_sgvec);

4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076
/* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
 * sglist without mark the sg which contain last skb data as the end.
 * So the caller can mannipulate sg list as will when padding new data after
 * the first call without calling sg_unmark_end to expend sg list.
 *
 * Scenario to use skb_to_sgvec_nomark:
 * 1. sg_init_table
 * 2. skb_to_sgvec_nomark(payload1)
 * 3. skb_to_sgvec_nomark(payload2)
 *
 * This is equivalent to:
 * 1. sg_init_table
 * 2. skb_to_sgvec(payload1)
 * 3. sg_unmark_end
 * 4. skb_to_sgvec(payload2)
 *
 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
 * is more preferable.
 */
int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
			int offset, int len)
{
4077
	return __skb_to_sgvec(skb, sg, offset, len, 0);
4078 4079 4080
}
EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);

4081 4082


4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114
/**
 *	skb_cow_data - Check that a socket buffer's data buffers are writable
 *	@skb: The socket buffer to check.
 *	@tailbits: Amount of trailing space to be added
 *	@trailer: Returned pointer to the skb where the @tailbits space begins
 *
 *	Make sure that the data buffers attached to a socket buffer are
 *	writable. If they are not, private copies are made of the data buffers
 *	and the socket buffer is set to use these instead.
 *
 *	If @tailbits is given, make sure that there is space to write @tailbits
 *	bytes of data beyond current end of socket buffer.  @trailer will be
 *	set to point to the skb in which this space begins.
 *
 *	The number of scatterlist elements required to completely map the
 *	COW'd and extended socket buffer will be returned.
 */
int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
{
	int copyflag;
	int elt;
	struct sk_buff *skb1, **skb_p;

	/* If skb is cloned or its head is paged, reallocate
	 * head pulling out all the pages (pages are considered not writable
	 * at the moment even if they are anonymous).
	 */
	if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
	    __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
		return -ENOMEM;

	/* Easy case. Most of packets will go this way. */
4115
	if (!skb_has_frag_list(skb)) {
4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149
		/* A little of trouble, not enough of space for trailer.
		 * This should not happen, when stack is tuned to generate
		 * good frames. OK, on miss we reallocate and reserve even more
		 * space, 128 bytes is fair. */

		if (skb_tailroom(skb) < tailbits &&
		    pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
			return -ENOMEM;

		/* Voila! */
		*trailer = skb;
		return 1;
	}

	/* Misery. We are in troubles, going to mincer fragments... */

	elt = 1;
	skb_p = &skb_shinfo(skb)->frag_list;
	copyflag = 0;

	while ((skb1 = *skb_p) != NULL) {
		int ntail = 0;

		/* The fragment is partially pulled by someone,
		 * this can happen on input. Copy it and everything
		 * after it. */

		if (skb_shared(skb1))
			copyflag = 1;

		/* If the skb is the last, worry about trailer. */

		if (skb1->next == NULL && tailbits) {
			if (skb_shinfo(skb1)->nr_frags ||
4150
			    skb_has_frag_list(skb1) ||
4151 4152 4153 4154 4155 4156 4157 4158
			    skb_tailroom(skb1) < tailbits)
				ntail = tailbits + 128;
		}

		if (copyflag ||
		    skb_cloned(skb1) ||
		    ntail ||
		    skb_shinfo(skb1)->nr_frags ||
4159
		    skb_has_frag_list(skb1)) {
4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190
			struct sk_buff *skb2;

			/* Fuck, we are miserable poor guys... */
			if (ntail == 0)
				skb2 = skb_copy(skb1, GFP_ATOMIC);
			else
				skb2 = skb_copy_expand(skb1,
						       skb_headroom(skb1),
						       ntail,
						       GFP_ATOMIC);
			if (unlikely(skb2 == NULL))
				return -ENOMEM;

			if (skb1->sk)
				skb_set_owner_w(skb2, skb1->sk);

			/* Looking around. Are we still alive?
			 * OK, link new skb, drop old one */

			skb2->next = skb1->next;
			*skb_p = skb2;
			kfree_skb(skb1);
			skb1 = skb2;
		}
		elt++;
		*trailer = skb1;
		skb_p = &skb1->next;
	}

	return elt;
}
4191
EXPORT_SYMBOL_GPL(skb_cow_data);
4192

4193 4194 4195 4196 4197 4198 4199
static void sock_rmem_free(struct sk_buff *skb)
{
	struct sock *sk = skb->sk;

	atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
}

4200 4201 4202 4203 4204 4205 4206 4207 4208
static void skb_set_err_queue(struct sk_buff *skb)
{
	/* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
	 * So, it is safe to (mis)use it to mark skbs on the error queue.
	 */
	skb->pkt_type = PACKET_OUTGOING;
	BUILD_BUG_ON(PACKET_OUTGOING == 0);
}

4209 4210 4211 4212 4213 4214
/*
 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
 */
int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
{
	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4215
	    (unsigned int)sk->sk_rcvbuf)
4216 4217 4218 4219 4220 4221
		return -ENOMEM;

	skb_orphan(skb);
	skb->sk = sk;
	skb->destructor = sock_rmem_free;
	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4222
	skb_set_err_queue(skb);
4223

4224 4225 4226
	/* before exiting rcu section, make sure dst is refcounted */
	skb_dst_force(skb);

4227 4228
	skb_queue_tail(&sk->sk_error_queue, skb);
	if (!sock_flag(sk, SOCK_DEAD))
4229
		sk->sk_error_report(sk);
4230 4231 4232 4233
	return 0;
}
EXPORT_SYMBOL(sock_queue_err_skb);

4234 4235 4236 4237 4238 4239
static bool is_icmp_err_skb(const struct sk_buff *skb)
{
	return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
		       SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
}

4240 4241 4242
struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
{
	struct sk_buff_head *q = &sk->sk_error_queue;
4243 4244
	struct sk_buff *skb, *skb_next = NULL;
	bool icmp_next = false;
4245
	unsigned long flags;
4246

4247
	spin_lock_irqsave(&q->lock, flags);
4248
	skb = __skb_dequeue(q);
4249
	if (skb && (skb_next = skb_peek(q))) {
4250
		icmp_next = is_icmp_err_skb(skb_next);
4251 4252 4253
		if (icmp_next)
			sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin;
	}
4254
	spin_unlock_irqrestore(&q->lock, flags);
4255

4256 4257 4258 4259
	if (is_icmp_err_skb(skb) && !icmp_next)
		sk->sk_err = 0;

	if (skb_next)
4260 4261 4262 4263 4264 4265
		sk->sk_error_report(sk);

	return skb;
}
EXPORT_SYMBOL(sock_dequeue_err_skb);

4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278
/**
 * skb_clone_sk - create clone of skb, and take reference to socket
 * @skb: the skb to clone
 *
 * This function creates a clone of a buffer that holds a reference on
 * sk_refcnt.  Buffers created via this function are meant to be
 * returned using sock_queue_err_skb, or free via kfree_skb.
 *
 * When passing buffers allocated with this function to sock_queue_err_skb
 * it is necessary to wrap the call with sock_hold/sock_put in order to
 * prevent the socket from being released prior to being enqueued on
 * the sk_error_queue.
 */
4279 4280 4281 4282 4283
struct sk_buff *skb_clone_sk(struct sk_buff *skb)
{
	struct sock *sk = skb->sk;
	struct sk_buff *clone;

4284
	if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299
		return NULL;

	clone = skb_clone(skb, GFP_ATOMIC);
	if (!clone) {
		sock_put(sk);
		return NULL;
	}

	clone->sk = sk;
	clone->destructor = sock_efree;

	return clone;
}
EXPORT_SYMBOL(skb_clone_sk);

4300 4301
static void __skb_complete_tx_timestamp(struct sk_buff *skb,
					struct sock *sk,
4302 4303
					int tstype,
					bool opt_stats)
4304 4305 4306 4307
{
	struct sock_exterr_skb *serr;
	int err;

4308 4309
	BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));

4310 4311 4312 4313
	serr = SKB_EXT_ERR(skb);
	memset(serr, 0, sizeof(*serr));
	serr->ee.ee_errno = ENOMSG;
	serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4314
	serr->ee.ee_info = tstype;
4315
	serr->opt_stats = opt_stats;
4316
	serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4317
	if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4318
		serr->ee.ee_data = skb_shinfo(skb)->tskey;
4319 4320
		if (sk->sk_protocol == IPPROTO_TCP &&
		    sk->sk_type == SOCK_STREAM)
4321 4322
			serr->ee.ee_data -= sk->sk_tskey;
	}
4323

4324
	err = sock_queue_err_skb(sk, skb);
4325

4326 4327 4328
	if (err)
		kfree_skb(skb);
}
4329

4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343
static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
{
	bool ret;

	if (likely(sysctl_tstamp_allow_data || tsonly))
		return true;

	read_lock_bh(&sk->sk_callback_lock);
	ret = sk->sk_socket && sk->sk_socket->file &&
	      file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
	read_unlock_bh(&sk->sk_callback_lock);
	return ret;
}

4344 4345 4346 4347 4348
void skb_complete_tx_timestamp(struct sk_buff *skb,
			       struct skb_shared_hwtstamps *hwtstamps)
{
	struct sock *sk = skb->sk;

4349
	if (!skb_may_tx_timestamp(sk, false))
4350
		goto err;
4351

4352 4353 4354
	/* Take a reference to prevent skb_orphan() from freeing the socket,
	 * but only if the socket refcount is not zero.
	 */
4355
	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4356
		*skb_hwtstamps(skb) = *hwtstamps;
4357
		__skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
4358
		sock_put(sk);
4359
		return;
4360
	}
4361 4362 4363

err:
	kfree_skb(skb);
4364 4365 4366 4367 4368 4369 4370 4371
}
EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);

void __skb_tstamp_tx(struct sk_buff *orig_skb,
		     struct skb_shared_hwtstamps *hwtstamps,
		     struct sock *sk, int tstype)
{
	struct sk_buff *skb;
4372
	bool tsonly, opt_stats = false;
4373

4374 4375 4376
	if (!sk)
		return;

4377 4378 4379 4380
	if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
	    skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
		return;

4381 4382
	tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
	if (!skb_may_tx_timestamp(sk, tsonly))
4383 4384
		return;

4385 4386 4387 4388
	if (tsonly) {
#ifdef CONFIG_INET
		if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
		    sk->sk_protocol == IPPROTO_TCP &&
4389
		    sk->sk_type == SOCK_STREAM) {
4390
			skb = tcp_get_timestamping_opt_stats(sk);
4391 4392
			opt_stats = true;
		} else
4393 4394 4395
#endif
			skb = alloc_skb(0, GFP_ATOMIC);
	} else {
4396
		skb = skb_clone(orig_skb, GFP_ATOMIC);
4397
	}
4398 4399 4400
	if (!skb)
		return;

4401
	if (tsonly) {
4402 4403
		skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
					     SKBTX_ANY_TSTAMP;
4404 4405 4406 4407 4408 4409 4410 4411
		skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
	}

	if (hwtstamps)
		*skb_hwtstamps(skb) = *hwtstamps;
	else
		skb->tstamp = ktime_get_real();

4412
	__skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
4413
}
4414 4415 4416 4417 4418 4419 4420 4421
EXPORT_SYMBOL_GPL(__skb_tstamp_tx);

void skb_tstamp_tx(struct sk_buff *orig_skb,
		   struct skb_shared_hwtstamps *hwtstamps)
{
	return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
			       SCM_TSTAMP_SND);
}
4422 4423
EXPORT_SYMBOL_GPL(skb_tstamp_tx);

4424 4425 4426 4427
void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
{
	struct sock *sk = skb->sk;
	struct sock_exterr_skb *serr;
4428
	int err = 1;
4429 4430 4431 4432 4433 4434 4435 4436 4437

	skb->wifi_acked_valid = 1;
	skb->wifi_acked = acked;

	serr = SKB_EXT_ERR(skb);
	memset(serr, 0, sizeof(*serr));
	serr->ee.ee_errno = ENOMSG;
	serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;

4438 4439 4440
	/* Take a reference to prevent skb_orphan() from freeing the socket,
	 * but only if the socket refcount is not zero.
	 */
4441
	if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4442 4443 4444
		err = sock_queue_err_skb(sk, skb);
		sock_put(sk);
	}
4445 4446 4447 4448 4449
	if (err)
		kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);

4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463
/**
 * skb_partial_csum_set - set up and verify partial csum values for packet
 * @skb: the skb to set
 * @start: the number of bytes after skb->data to start checksumming.
 * @off: the offset from start to place the checksum.
 *
 * For untrusted partially-checksummed packets, we need to make sure the values
 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
 *
 * This function checks and sets those values and skb->ip_summed: if this
 * returns false you should drop the packet.
 */
bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
{
4464 4465 4466 4467 4468 4469
	u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
	u32 csum_start = skb_headroom(skb) + (u32)start;

	if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) {
		net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
				     start, off, skb_headroom(skb), skb_headlen(skb));
4470 4471 4472
		return false;
	}
	skb->ip_summed = CHECKSUM_PARTIAL;
4473
	skb->csum_start = csum_start;
4474
	skb->csum_offset = off;
4475
	skb_set_transport_header(skb, start);
4476 4477
	return true;
}
4478
EXPORT_SYMBOL_GPL(skb_partial_csum_set);
4479

P
Paul Durrant 已提交
4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500
static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
			       unsigned int max)
{
	if (skb_headlen(skb) >= len)
		return 0;

	/* If we need to pullup then pullup to the max, so we
	 * won't need to do it again.
	 */
	if (max > skb->len)
		max = skb->len;

	if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
		return -ENOMEM;

	if (skb_headlen(skb) < len)
		return -EPROTO;

	return 0;
}

4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531
#define MAX_TCP_HDR_LEN (15 * 4)

static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
				      typeof(IPPROTO_IP) proto,
				      unsigned int off)
{
	switch (proto) {
		int err;

	case IPPROTO_TCP:
		err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
					  off + MAX_TCP_HDR_LEN);
		if (!err && !skb_partial_csum_set(skb, off,
						  offsetof(struct tcphdr,
							   check)))
			err = -EPROTO;
		return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;

	case IPPROTO_UDP:
		err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
					  off + sizeof(struct udphdr));
		if (!err && !skb_partial_csum_set(skb, off,
						  offsetof(struct udphdr,
							   check)))
			err = -EPROTO;
		return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
	}

	return ERR_PTR(-EPROTO);
}

P
Paul Durrant 已提交
4532 4533 4534 4535 4536
/* This value should be large enough to cover a tagged ethernet header plus
 * maximally sized IP and TCP or UDP headers.
 */
#define MAX_IP_HDR_LEN 128

4537
static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
P
Paul Durrant 已提交
4538 4539 4540
{
	unsigned int off;
	bool fragment;
4541
	__sum16 *csum;
P
Paul Durrant 已提交
4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561
	int err;

	fragment = false;

	err = skb_maybe_pull_tail(skb,
				  sizeof(struct iphdr),
				  MAX_IP_HDR_LEN);
	if (err < 0)
		goto out;

	if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
		fragment = true;

	off = ip_hdrlen(skb);

	err = -EPROTO;

	if (fragment)
		goto out;

4562 4563 4564
	csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
	if (IS_ERR(csum))
		return PTR_ERR(csum);
P
Paul Durrant 已提交
4565

4566 4567 4568 4569 4570
	if (recalculate)
		*csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
					   ip_hdr(skb)->daddr,
					   skb->len - off,
					   ip_hdr(skb)->protocol, 0);
P
Paul Durrant 已提交
4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592
	err = 0;

out:
	return err;
}

/* This value should be large enough to cover a tagged ethernet header plus
 * an IPv6 header, all options, and a maximal TCP or UDP header.
 */
#define MAX_IPV6_HDR_LEN 256

#define OPT_HDR(type, skb, off) \
	(type *)(skb_network_header(skb) + (off))

static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
{
	int err;
	u8 nexthdr;
	unsigned int off;
	unsigned int len;
	bool fragment;
	bool done;
4593
	__sum16 *csum;
P
Paul Durrant 已提交
4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670

	fragment = false;
	done = false;

	off = sizeof(struct ipv6hdr);

	err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
	if (err < 0)
		goto out;

	nexthdr = ipv6_hdr(skb)->nexthdr;

	len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
	while (off <= len && !done) {
		switch (nexthdr) {
		case IPPROTO_DSTOPTS:
		case IPPROTO_HOPOPTS:
		case IPPROTO_ROUTING: {
			struct ipv6_opt_hdr *hp;

			err = skb_maybe_pull_tail(skb,
						  off +
						  sizeof(struct ipv6_opt_hdr),
						  MAX_IPV6_HDR_LEN);
			if (err < 0)
				goto out;

			hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
			nexthdr = hp->nexthdr;
			off += ipv6_optlen(hp);
			break;
		}
		case IPPROTO_AH: {
			struct ip_auth_hdr *hp;

			err = skb_maybe_pull_tail(skb,
						  off +
						  sizeof(struct ip_auth_hdr),
						  MAX_IPV6_HDR_LEN);
			if (err < 0)
				goto out;

			hp = OPT_HDR(struct ip_auth_hdr, skb, off);
			nexthdr = hp->nexthdr;
			off += ipv6_authlen(hp);
			break;
		}
		case IPPROTO_FRAGMENT: {
			struct frag_hdr *hp;

			err = skb_maybe_pull_tail(skb,
						  off +
						  sizeof(struct frag_hdr),
						  MAX_IPV6_HDR_LEN);
			if (err < 0)
				goto out;

			hp = OPT_HDR(struct frag_hdr, skb, off);

			if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
				fragment = true;

			nexthdr = hp->nexthdr;
			off += sizeof(struct frag_hdr);
			break;
		}
		default:
			done = true;
			break;
		}
	}

	err = -EPROTO;

	if (!done || fragment)
		goto out;

4671 4672 4673
	csum = skb_checksum_setup_ip(skb, nexthdr, off);
	if (IS_ERR(csum))
		return PTR_ERR(csum);
P
Paul Durrant 已提交
4674

4675 4676 4677 4678
	if (recalculate)
		*csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
					 &ipv6_hdr(skb)->daddr,
					 skb->len - off, nexthdr, 0);
P
Paul Durrant 已提交
4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695
	err = 0;

out:
	return err;
}

/**
 * skb_checksum_setup - set up partial checksum offset
 * @skb: the skb to set up
 * @recalculate: if true the pseudo-header checksum will be recalculated
 */
int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
{
	int err;

	switch (skb->protocol) {
	case htons(ETH_P_IP):
4696
		err = skb_checksum_setup_ipv4(skb, recalculate);
P
Paul Durrant 已提交
4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711
		break;

	case htons(ETH_P_IPV6):
		err = skb_checksum_setup_ipv6(skb, recalculate);
		break;

	default:
		err = -EPROTO;
		break;
	}

	return err;
}
EXPORT_SYMBOL(skb_checksum_setup);

4712 4713 4714 4715 4716 4717 4718 4719 4720 4721
/**
 * skb_checksum_maybe_trim - maybe trims the given skb
 * @skb: the skb to check
 * @transport_len: the data length beyond the network header
 *
 * Checks whether the given skb has data beyond the given transport length.
 * If so, returns a cloned skb trimmed to this transport length.
 * Otherwise returns the provided skb. Returns NULL in error cases
 * (e.g. transport_len exceeds skb length or out-of-memory).
 *
4722 4723
 * Caller needs to set the skb transport header and free any returned skb if it
 * differs from the provided skb.
4724 4725 4726 4727 4728 4729 4730 4731
 */
static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
					       unsigned int transport_len)
{
	struct sk_buff *skb_chk;
	unsigned int len = skb_transport_offset(skb) + transport_len;
	int ret;

4732
	if (skb->len < len)
4733
		return NULL;
4734
	else if (skb->len == len)
4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761
		return skb;

	skb_chk = skb_clone(skb, GFP_ATOMIC);
	if (!skb_chk)
		return NULL;

	ret = pskb_trim_rcsum(skb_chk, len);
	if (ret) {
		kfree_skb(skb_chk);
		return NULL;
	}

	return skb_chk;
}

/**
 * skb_checksum_trimmed - validate checksum of an skb
 * @skb: the skb to check
 * @transport_len: the data length beyond the network header
 * @skb_chkf: checksum function to use
 *
 * Applies the given checksum function skb_chkf to the provided skb.
 * Returns a checked and maybe trimmed skb. Returns NULL on error.
 *
 * If the skb has data beyond the given transport length, then a
 * trimmed & cloned skb is checked and returned.
 *
4762 4763
 * Caller needs to set the skb transport header and free any returned skb if it
 * differs from the provided skb.
4764 4765 4766 4767 4768 4769 4770
 */
struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
				     unsigned int transport_len,
				     __sum16(*skb_chkf)(struct sk_buff *skb))
{
	struct sk_buff *skb_chk;
	unsigned int offset = skb_transport_offset(skb);
4771
	__sum16 ret;
4772 4773 4774

	skb_chk = skb_checksum_maybe_trim(skb, transport_len);
	if (!skb_chk)
4775
		goto err;
4776

4777 4778
	if (!pskb_may_pull(skb_chk, offset))
		goto err;
4779

4780
	skb_pull_rcsum(skb_chk, offset);
4781
	ret = skb_chkf(skb_chk);
4782
	skb_push_rcsum(skb_chk, offset);
4783

4784 4785
	if (ret)
		goto err;
4786 4787

	return skb_chk;
4788 4789 4790 4791 4792 4793 4794

err:
	if (skb_chk && skb_chk != skb)
		kfree_skb(skb_chk);

	return NULL;

4795 4796 4797
}
EXPORT_SYMBOL(skb_checksum_trimmed);

4798 4799
void __skb_warn_lro_forwarding(const struct sk_buff *skb)
{
4800 4801
	net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
			     skb->dev->name);
4802 4803
}
EXPORT_SYMBOL(__skb_warn_lro_forwarding);
E
Eric Dumazet 已提交
4804 4805 4806

void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
{
E
Eric Dumazet 已提交
4807 4808
	if (head_stolen) {
		skb_release_head_state(skb);
E
Eric Dumazet 已提交
4809
		kmem_cache_free(skbuff_head_cache, skb);
E
Eric Dumazet 已提交
4810
	} else {
E
Eric Dumazet 已提交
4811
		__kfree_skb(skb);
E
Eric Dumazet 已提交
4812
	}
E
Eric Dumazet 已提交
4813 4814 4815 4816 4817 4818 4819 4820
}
EXPORT_SYMBOL(kfree_skb_partial);

/**
 * skb_try_coalesce - try to merge skb to prior one
 * @to: prior buffer
 * @from: buffer to add
 * @fragstolen: pointer to boolean
R
Randy Dunlap 已提交
4821
 * @delta_truesize: how much more was allocated than was requested
E
Eric Dumazet 已提交
4822 4823 4824 4825
 */
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
		      bool *fragstolen, int *delta_truesize)
{
4826
	struct skb_shared_info *to_shinfo, *from_shinfo;
E
Eric Dumazet 已提交
4827 4828 4829 4830 4831 4832 4833 4834
	int i, delta, len = from->len;

	*fragstolen = false;

	if (skb_cloned(to))
		return false;

	if (len <= skb_tailroom(to)) {
4835 4836
		if (len)
			BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
E
Eric Dumazet 已提交
4837 4838 4839 4840
		*delta_truesize = 0;
		return true;
	}

4841 4842 4843
	to_shinfo = skb_shinfo(to);
	from_shinfo = skb_shinfo(from);
	if (to_shinfo->frag_list || from_shinfo->frag_list)
E
Eric Dumazet 已提交
4844
		return false;
W
Willem de Bruijn 已提交
4845 4846
	if (skb_zcopy(to) || skb_zcopy(from))
		return false;
E
Eric Dumazet 已提交
4847 4848 4849 4850 4851

	if (skb_headlen(from) != 0) {
		struct page *page;
		unsigned int offset;

4852 4853
		if (to_shinfo->nr_frags +
		    from_shinfo->nr_frags >= MAX_SKB_FRAGS)
E
Eric Dumazet 已提交
4854 4855 4856 4857 4858 4859 4860 4861 4862 4863
			return false;

		if (skb_head_is_locked(from))
			return false;

		delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));

		page = virt_to_head_page(from->head);
		offset = from->data - (unsigned char *)page_address(page);

4864
		skb_fill_page_desc(to, to_shinfo->nr_frags,
E
Eric Dumazet 已提交
4865 4866 4867
				   page, offset, skb_headlen(from));
		*fragstolen = true;
	} else {
4868 4869
		if (to_shinfo->nr_frags +
		    from_shinfo->nr_frags > MAX_SKB_FRAGS)
E
Eric Dumazet 已提交
4870 4871
			return false;

4872
		delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
E
Eric Dumazet 已提交
4873 4874 4875 4876
	}

	WARN_ON_ONCE(delta < len);

4877 4878 4879 4880
	memcpy(to_shinfo->frags + to_shinfo->nr_frags,
	       from_shinfo->frags,
	       from_shinfo->nr_frags * sizeof(skb_frag_t));
	to_shinfo->nr_frags += from_shinfo->nr_frags;
E
Eric Dumazet 已提交
4881 4882

	if (!skb_cloned(from))
4883
		from_shinfo->nr_frags = 0;
E
Eric Dumazet 已提交
4884

4885 4886 4887
	/* if the skb is not cloned this does nothing
	 * since we set nr_frags to 0.
	 */
4888 4889
	for (i = 0; i < from_shinfo->nr_frags; i++)
		__skb_frag_ref(&from_shinfo->frags[i]);
E
Eric Dumazet 已提交
4890 4891 4892 4893 4894 4895 4896 4897 4898

	to->truesize += delta;
	to->len += len;
	to->data_len += len;

	*delta_truesize = delta;
	return true;
}
EXPORT_SYMBOL(skb_try_coalesce);
4899 4900

/**
4901
 * skb_scrub_packet - scrub an skb
4902 4903
 *
 * @skb: buffer to clean
4904 4905 4906 4907 4908 4909 4910 4911
 * @xnet: packet is crossing netns
 *
 * skb_scrub_packet can be used after encapsulating or decapsulting a packet
 * into/from a tunnel. Some information have to be cleared during these
 * operations.
 * skb_scrub_packet can also be used to clean a skb before injecting it in
 * another namespace (@xnet == true). We have to clear all information in the
 * skb that could impact namespace isolation.
4912
 */
4913
void skb_scrub_packet(struct sk_buff *skb, bool xnet)
4914 4915 4916
{
	skb->pkt_type = PACKET_HOST;
	skb->skb_iif = 0;
W
WANG Cong 已提交
4917
	skb->ignore_df = 0;
4918 4919 4920 4921
	skb_dst_drop(skb);
	secpath_reset(skb);
	nf_reset(skb);
	nf_reset_trace(skb);
4922

4923 4924
#ifdef CONFIG_NET_SWITCHDEV
	skb->offload_fwd_mark = 0;
4925
	skb->offload_l3_fwd_mark = 0;
4926 4927
#endif

4928 4929 4930
	if (!xnet)
		return;

4931
	ipvs_reset(skb);
4932
	skb->mark = 0;
4933
	skb->tstamp = 0;
4934 4935
}
EXPORT_SYMBOL_GPL(skb_scrub_packet);
4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946

/**
 * skb_gso_transport_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_transport_seglen is used to determine the real size of the
 * individual segments, including Layer4 headers (TCP/UDP).
 *
 * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
 */
4947
static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
4948 4949
{
	const struct skb_shared_info *shinfo = skb_shinfo(skb);
4950
	unsigned int thlen = 0;
4951

4952 4953 4954
	if (skb->encapsulation) {
		thlen = skb_inner_transport_header(skb) -
			skb_transport_header(skb);
4955

4956 4957 4958 4959
		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
			thlen += inner_tcp_hdrlen(skb);
	} else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
		thlen = tcp_hdrlen(skb);
4960
	} else if (unlikely(skb_is_gso_sctp(skb))) {
M
Marcelo Ricardo Leitner 已提交
4961
		thlen = sizeof(struct sctphdr);
W
Willem de Bruijn 已提交
4962 4963
	} else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
		thlen = sizeof(struct udphdr);
4964
	}
4965 4966 4967 4968
	/* UFO sets gso_size to the size of the fragmentation
	 * payload, i.e. the size of the L4 (UDP) header is already
	 * accounted for.
	 */
4969
	return thlen + shinfo->gso_size;
4970
}
4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004

/**
 * skb_gso_network_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_network_seglen is used to determine the real size of the
 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
 *
 * The MAC/L2 header is not accounted for.
 */
static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
{
	unsigned int hdr_len = skb_transport_header(skb) -
			       skb_network_header(skb);

	return hdr_len + skb_gso_transport_seglen(skb);
}

/**
 * skb_gso_mac_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_mac_seglen is used to determine the real size of the
 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
 * headers (TCP/UDP).
 */
static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
{
	unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);

	return hdr_len + skb_gso_transport_seglen(skb);
}
5005

5006
/**
5007
 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
5008
 *
5009 5010
 * There are a couple of instances where we have a GSO skb, and we
 * want to determine what size it would be after it is segmented.
5011
 *
5012 5013 5014 5015 5016 5017
 * We might want to check:
 * -    L3+L4+payload size (e.g. IP forwarding)
 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
 *
 * This is a helper to do that correctly considering GSO_BY_FRAGS.
 *
5018 5019
 * @skb: GSO skb
 *
5020 5021 5022 5023 5024 5025
 * @seg_len: The segmented length (from skb_gso_*_seglen). In the
 *           GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
 *
 * @max_len: The maximum permissible length.
 *
 * Returns true if the segmented length <= max length.
5026
 */
5027 5028 5029
static inline bool skb_gso_size_check(const struct sk_buff *skb,
				      unsigned int seg_len,
				      unsigned int max_len) {
5030 5031 5032 5033
	const struct skb_shared_info *shinfo = skb_shinfo(skb);
	const struct sk_buff *iter;

	if (shinfo->gso_size != GSO_BY_FRAGS)
5034
		return seg_len <= max_len;
5035 5036

	/* Undo this so we can re-use header sizes */
5037
	seg_len -= GSO_BY_FRAGS;
5038 5039

	skb_walk_frags(skb, iter) {
5040
		if (seg_len + skb_headlen(iter) > max_len)
5041 5042 5043 5044 5045
			return false;
	}

	return true;
}
5046 5047

/**
5048
 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
5049 5050 5051 5052
 *
 * @skb: GSO skb
 * @mtu: MTU to validate against
 *
5053 5054 5055
 * skb_gso_validate_network_len validates if a given skb will fit a
 * wanted MTU once split. It considers L3 headers, L4 headers, and the
 * payload.
5056
 */
5057
bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
5058 5059 5060
{
	return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
}
5061
EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
5062

5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077
/**
 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
 *
 * @skb: GSO skb
 * @len: length to validate against
 *
 * skb_gso_validate_mac_len validates if a given skb will fit a wanted
 * length once split, including L2, L3 and L4 headers and the payload.
 */
bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
{
	return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
}
EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);

5078 5079
static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
{
5080 5081
	int mac_len;

5082 5083 5084 5085 5086
	if (skb_cow(skb, skb_headroom(skb)) < 0) {
		kfree_skb(skb);
		return NULL;
	}

5087
	mac_len = skb->data - skb_mac_header(skb);
5088 5089 5090 5091
	if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
		memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
			mac_len - VLAN_HLEN - ETH_TLEN);
	}
5092 5093 5094 5095 5096 5097 5098 5099 5100
	skb->mac_header += VLAN_HLEN;
	return skb;
}

struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
{
	struct vlan_hdr *vhdr;
	u16 vlan_tci;

5101
	if (unlikely(skb_vlan_tag_present(skb))) {
5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134
		/* vlan_tci is already set-up so leave this for another time */
		return skb;
	}

	skb = skb_share_check(skb, GFP_ATOMIC);
	if (unlikely(!skb))
		goto err_free;

	if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
		goto err_free;

	vhdr = (struct vlan_hdr *)skb->data;
	vlan_tci = ntohs(vhdr->h_vlan_TCI);
	__vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);

	skb_pull_rcsum(skb, VLAN_HLEN);
	vlan_set_encap_proto(skb, vhdr);

	skb = skb_reorder_vlan_header(skb);
	if (unlikely(!skb))
		goto err_free;

	skb_reset_network_header(skb);
	skb_reset_transport_header(skb);
	skb_reset_mac_len(skb);

	return skb;

err_free:
	kfree_skb(skb);
	return NULL;
}
EXPORT_SYMBOL(skb_vlan_untag);
5135

5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147
int skb_ensure_writable(struct sk_buff *skb, int write_len)
{
	if (!pskb_may_pull(skb, write_len))
		return -ENOMEM;

	if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
		return 0;

	return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
}
EXPORT_SYMBOL(skb_ensure_writable);

5148 5149 5150 5151
/* remove VLAN header from packet and update csum accordingly.
 * expects a non skb_vlan_tag_present skb with a vlan tag payload
 */
int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
5152 5153
{
	struct vlan_hdr *vhdr;
5154
	int offset = skb->data - skb_mac_header(skb);
5155 5156
	int err;

5157 5158 5159 5160 5161 5162
	if (WARN_ONCE(offset,
		      "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
		      offset)) {
		return -EINVAL;
	}

5163 5164
	err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
	if (unlikely(err))
5165
		return err;
5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184

	skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);

	vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
	*vlan_tci = ntohs(vhdr->h_vlan_TCI);

	memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
	__skb_pull(skb, VLAN_HLEN);

	vlan_set_encap_proto(skb, vhdr);
	skb->mac_header += VLAN_HLEN;

	if (skb_network_offset(skb) < ETH_HLEN)
		skb_set_network_header(skb, ETH_HLEN);

	skb_reset_mac_len(skb);

	return err;
}
5185
EXPORT_SYMBOL(__skb_vlan_pop);
5186

5187 5188 5189
/* Pop a vlan tag either from hwaccel or from payload.
 * Expects skb->data at mac header.
 */
5190 5191 5192 5193 5194 5195
int skb_vlan_pop(struct sk_buff *skb)
{
	u16 vlan_tci;
	__be16 vlan_proto;
	int err;

5196
	if (likely(skb_vlan_tag_present(skb))) {
5197
		__vlan_hwaccel_clear_tag(skb);
5198
	} else {
5199
		if (unlikely(!eth_type_vlan(skb->protocol)))
5200 5201 5202 5203 5204 5205 5206
			return 0;

		err = __skb_vlan_pop(skb, &vlan_tci);
		if (err)
			return err;
	}
	/* move next vlan tag to hw accel tag */
5207
	if (likely(!eth_type_vlan(skb->protocol)))
5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219
		return 0;

	vlan_proto = skb->protocol;
	err = __skb_vlan_pop(skb, &vlan_tci);
	if (unlikely(err))
		return err;

	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
	return 0;
}
EXPORT_SYMBOL(skb_vlan_pop);

5220 5221 5222
/* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
 * Expects skb->data at mac header.
 */
5223 5224
int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
{
5225
	if (skb_vlan_tag_present(skb)) {
5226
		int offset = skb->data - skb_mac_header(skb);
5227 5228
		int err;

5229 5230 5231 5232 5233 5234
		if (WARN_ONCE(offset,
			      "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
			      offset)) {
			return -EINVAL;
		}

5235
		err = __vlan_insert_tag(skb, skb->vlan_proto,
5236
					skb_vlan_tag_get(skb));
5237
		if (err)
5238
			return err;
5239

5240 5241 5242
		skb->protocol = skb->vlan_proto;
		skb->mac_len += VLAN_HLEN;

5243
		skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5244 5245 5246 5247 5248 5249
	}
	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
	return 0;
}
EXPORT_SYMBOL(skb_vlan_push);

5250 5251 5252
/**
 * alloc_skb_with_frags - allocate skb with page frags
 *
5253 5254 5255 5256 5257
 * @header_len: size of linear part
 * @data_len: needed length in frags
 * @max_page_order: max page order desired.
 * @errcode: pointer to error code if any
 * @gfp_mask: allocation mask
5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281
 *
 * This can be used to allocate a paged skb, given a maximal order for frags.
 */
struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
				     unsigned long data_len,
				     int max_page_order,
				     int *errcode,
				     gfp_t gfp_mask)
{
	int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	unsigned long chunk;
	struct sk_buff *skb;
	struct page *page;
	gfp_t gfp_head;
	int i;

	*errcode = -EMSGSIZE;
	/* Note this test could be relaxed, if we succeed to allocate
	 * high order pages...
	 */
	if (npages > MAX_SKB_FRAGS)
		return NULL;

	gfp_head = gfp_mask;
5282
	if (gfp_head & __GFP_DIRECT_RECLAIM)
5283
		gfp_head |= __GFP_RETRY_MAYFAIL;
5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296

	*errcode = -ENOBUFS;
	skb = alloc_skb(header_len, gfp_head);
	if (!skb)
		return NULL;

	skb->truesize += npages << PAGE_SHIFT;

	for (i = 0; npages > 0; i++) {
		int order = max_page_order;

		while (order) {
			if (npages >= 1 << order) {
5297
				page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
5298
						   __GFP_COMP |
5299
						   __GFP_NOWARN,
5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325
						   order);
				if (page)
					goto fill_page;
				/* Do not retry other high order allocations */
				order = 1;
				max_page_order = 0;
			}
			order--;
		}
		page = alloc_page(gfp_mask);
		if (!page)
			goto failure;
fill_page:
		chunk = min_t(unsigned long, data_len,
			      PAGE_SIZE << order);
		skb_fill_page_desc(skb, i, page, 0, chunk);
		data_len -= chunk;
		npages -= 1 << order;
	}
	return skb;

failure:
	kfree_skb(skb);
	return NULL;
}
EXPORT_SYMBOL(alloc_skb_with_frags);
5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561

/* carve out the first off bytes from skb when off < headlen */
static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
				    const int headlen, gfp_t gfp_mask)
{
	int i;
	int size = skb_end_offset(skb);
	int new_hlen = headlen - off;
	u8 *data;

	size = SKB_DATA_ALIGN(size);

	if (skb_pfmemalloc(skb))
		gfp_mask |= __GFP_MEMALLOC;
	data = kmalloc_reserve(size +
			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
			       gfp_mask, NUMA_NO_NODE, NULL);
	if (!data)
		return -ENOMEM;

	size = SKB_WITH_OVERHEAD(ksize(data));

	/* Copy real data, and all frags */
	skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
	skb->len -= off;

	memcpy((struct skb_shared_info *)(data + size),
	       skb_shinfo(skb),
	       offsetof(struct skb_shared_info,
			frags[skb_shinfo(skb)->nr_frags]));
	if (skb_cloned(skb)) {
		/* drop the old head gracefully */
		if (skb_orphan_frags(skb, gfp_mask)) {
			kfree(data);
			return -ENOMEM;
		}
		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
			skb_frag_ref(skb, i);
		if (skb_has_frag_list(skb))
			skb_clone_fraglist(skb);
		skb_release_data(skb);
	} else {
		/* we can reuse existing recount- all we did was
		 * relocate values
		 */
		skb_free_head(skb);
	}

	skb->head = data;
	skb->data = data;
	skb->head_frag = 0;
#ifdef NET_SKBUFF_DATA_USES_OFFSET
	skb->end = size;
#else
	skb->end = skb->head + size;
#endif
	skb_set_tail_pointer(skb, skb_headlen(skb));
	skb_headers_offset_update(skb, 0);
	skb->cloned = 0;
	skb->hdr_len = 0;
	skb->nohdr = 0;
	atomic_set(&skb_shinfo(skb)->dataref, 1);

	return 0;
}

static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);

/* carve out the first eat bytes from skb's frag_list. May recurse into
 * pskb_carve()
 */
static int pskb_carve_frag_list(struct sk_buff *skb,
				struct skb_shared_info *shinfo, int eat,
				gfp_t gfp_mask)
{
	struct sk_buff *list = shinfo->frag_list;
	struct sk_buff *clone = NULL;
	struct sk_buff *insp = NULL;

	do {
		if (!list) {
			pr_err("Not enough bytes to eat. Want %d\n", eat);
			return -EFAULT;
		}
		if (list->len <= eat) {
			/* Eaten as whole. */
			eat -= list->len;
			list = list->next;
			insp = list;
		} else {
			/* Eaten partially. */
			if (skb_shared(list)) {
				clone = skb_clone(list, gfp_mask);
				if (!clone)
					return -ENOMEM;
				insp = list->next;
				list = clone;
			} else {
				/* This may be pulled without problems. */
				insp = list;
			}
			if (pskb_carve(list, eat, gfp_mask) < 0) {
				kfree_skb(clone);
				return -ENOMEM;
			}
			break;
		}
	} while (eat);

	/* Free pulled out fragments. */
	while ((list = shinfo->frag_list) != insp) {
		shinfo->frag_list = list->next;
		kfree_skb(list);
	}
	/* And insert new clone at head. */
	if (clone) {
		clone->next = list;
		shinfo->frag_list = clone;
	}
	return 0;
}

/* carve off first len bytes from skb. Split line (off) is in the
 * non-linear part of skb
 */
static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
				       int pos, gfp_t gfp_mask)
{
	int i, k = 0;
	int size = skb_end_offset(skb);
	u8 *data;
	const int nfrags = skb_shinfo(skb)->nr_frags;
	struct skb_shared_info *shinfo;

	size = SKB_DATA_ALIGN(size);

	if (skb_pfmemalloc(skb))
		gfp_mask |= __GFP_MEMALLOC;
	data = kmalloc_reserve(size +
			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
			       gfp_mask, NUMA_NO_NODE, NULL);
	if (!data)
		return -ENOMEM;

	size = SKB_WITH_OVERHEAD(ksize(data));

	memcpy((struct skb_shared_info *)(data + size),
	       skb_shinfo(skb), offsetof(struct skb_shared_info,
					 frags[skb_shinfo(skb)->nr_frags]));
	if (skb_orphan_frags(skb, gfp_mask)) {
		kfree(data);
		return -ENOMEM;
	}
	shinfo = (struct skb_shared_info *)(data + size);
	for (i = 0; i < nfrags; i++) {
		int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);

		if (pos + fsize > off) {
			shinfo->frags[k] = skb_shinfo(skb)->frags[i];

			if (pos < off) {
				/* Split frag.
				 * We have two variants in this case:
				 * 1. Move all the frag to the second
				 *    part, if it is possible. F.e.
				 *    this approach is mandatory for TUX,
				 *    where splitting is expensive.
				 * 2. Split is accurately. We make this.
				 */
				shinfo->frags[0].page_offset += off - pos;
				skb_frag_size_sub(&shinfo->frags[0], off - pos);
			}
			skb_frag_ref(skb, i);
			k++;
		}
		pos += fsize;
	}
	shinfo->nr_frags = k;
	if (skb_has_frag_list(skb))
		skb_clone_fraglist(skb);

	if (k == 0) {
		/* split line is in frag list */
		pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask);
	}
	skb_release_data(skb);

	skb->head = data;
	skb->head_frag = 0;
	skb->data = data;
#ifdef NET_SKBUFF_DATA_USES_OFFSET
	skb->end = size;
#else
	skb->end = skb->head + size;
#endif
	skb_reset_tail_pointer(skb);
	skb_headers_offset_update(skb, 0);
	skb->cloned   = 0;
	skb->hdr_len  = 0;
	skb->nohdr    = 0;
	skb->len -= off;
	skb->data_len = skb->len;
	atomic_set(&skb_shinfo(skb)->dataref, 1);
	return 0;
}

/* remove len bytes from the beginning of the skb */
static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
{
	int headlen = skb_headlen(skb);

	if (len < headlen)
		return pskb_carve_inside_header(skb, len, headlen, gfp);
	else
		return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
}

/* Extract to_copy bytes starting at off from skb, and return this in
 * a new skb
 */
struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
			     int to_copy, gfp_t gfp)
{
	struct sk_buff  *clone = skb_clone(skb, gfp);

	if (!clone)
		return NULL;

	if (pskb_carve(clone, off, gfp) < 0 ||
	    pskb_trim(clone, to_copy)) {
		kfree_skb(clone);
		return NULL;
	}
	return clone;
}
EXPORT_SYMBOL(pskb_extract);
5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576

/**
 * skb_condense - try to get rid of fragments/frag_list if possible
 * @skb: buffer
 *
 * Can be used to save memory before skb is added to a busy queue.
 * If packet has bytes in frags and enough tail room in skb->head,
 * pull all of them, so that we can free the frags right now and adjust
 * truesize.
 * Notes:
 *	We do not reallocate skb->head thus can not fail.
 *	Caller must re-evaluate skb->truesize if needed.
 */
void skb_condense(struct sk_buff *skb)
{
5577 5578 5579 5580
	if (skb->data_len) {
		if (skb->data_len > skb->end - skb->tail ||
		    skb_cloned(skb))
			return;
5581

5582 5583 5584 5585 5586 5587 5588 5589 5590
		/* Nice, we can free page frag(s) right now */
		__pskb_pull_tail(skb, skb->data_len);
	}
	/* At this point, skb->truesize might be over estimated,
	 * because skb had a fragment, and fragments do not tell
	 * their truesize.
	 * When we pulled its content into skb->head, fragment
	 * was freed, but __pskb_pull_tail() could not possibly
	 * adjust skb->truesize, not knowing the frag truesize.
5591 5592 5593
	 */
	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
}
5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708

#ifdef CONFIG_SKB_EXTENSIONS
static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
{
	return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
}

static struct skb_ext *skb_ext_alloc(void)
{
	struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);

	if (new) {
		memset(new->offset, 0, sizeof(new->offset));
		refcount_set(&new->refcnt, 1);
	}

	return new;
}

static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old)
{
	struct skb_ext *new;

	if (refcount_read(&old->refcnt) == 1)
		return old;

	new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
	if (!new)
		return NULL;

	memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
	refcount_set(&new->refcnt, 1);

	__skb_ext_put(old);
	return new;
}

/**
 * skb_ext_add - allocate space for given extension, COW if needed
 * @skb: buffer
 * @id: extension to allocate space for
 *
 * Allocates enough space for the given extension.
 * If the extension is already present, a pointer to that extension
 * is returned.
 *
 * If the skb was cloned, COW applies and the returned memory can be
 * modified without changing the extension space of clones buffers.
 *
 * Returns pointer to the extension or NULL on allocation failure.
 */
void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
{
	struct skb_ext *new, *old = NULL;
	unsigned int newlen, newoff;

	if (skb->active_extensions) {
		old = skb->extensions;

		new = skb_ext_maybe_cow(old);
		if (!new)
			return NULL;

		if (__skb_ext_exist(old, id)) {
			if (old != new)
				skb->extensions = new;
			goto set_active;
		}

		newoff = old->chunks;
	} else {
		newoff = SKB_EXT_CHUNKSIZEOF(*new);

		new = skb_ext_alloc();
		if (!new)
			return NULL;
	}

	newlen = newoff + skb_ext_type_len[id];
	new->chunks = newlen;
	new->offset[id] = newoff;
	skb->extensions = new;
set_active:
	skb->active_extensions |= 1 << id;
	return skb_ext_get_ptr(new, id);
}
EXPORT_SYMBOL(skb_ext_add);

void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
{
	struct skb_ext *ext = skb->extensions;

	skb->active_extensions &= ~(1 << id);
	if (skb->active_extensions == 0) {
		skb->extensions = NULL;
		__skb_ext_put(ext);
	}
}
EXPORT_SYMBOL(__skb_ext_del);

void __skb_ext_put(struct skb_ext *ext)
{
	/* If this is last clone, nothing can increment
	 * it after check passes.  Avoids one atomic op.
	 */
	if (refcount_read(&ext->refcnt) == 1)
		goto free_now;

	if (!refcount_dec_and_test(&ext->refcnt))
		return;
free_now:
	kmem_cache_free(skbuff_ext_cache, ext);
}
EXPORT_SYMBOL(__skb_ext_put);
#endif /* CONFIG_SKB_EXTENSIONS */