filter.h 30.2 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
L
Linus Torvalds 已提交
2 3 4 5 6 7
/*
 * Linux Socket Filter Data Structures
 */
#ifndef __LINUX_FILTER_H__
#define __LINUX_FILTER_H__

8 9
#include <stdarg.h>

10
#include <linux/atomic.h>
11
#include <linux/refcount.h>
12
#include <linux/compat.h>
13
#include <linux/skbuff.h>
14 15
#include <linux/linkage.h>
#include <linux/printk.h>
16
#include <linux/workqueue.h>
17
#include <linux/sched.h>
18
#include <linux/capability.h>
19
#include <linux/cryptohash.h>
20
#include <linux/set_memory.h>
21
#include <linux/kallsyms.h>
22
#include <linux/if_vlan.h>
23

24
#include <net/sch_generic.h>
25 26

#include <uapi/linux/filter.h>
27
#include <uapi/linux/bpf.h>
28 29 30 31

struct sk_buff;
struct sock;
struct seccomp_data;
32
struct bpf_prog_aux;
33
struct xdp_rxq_info;
34
struct xdp_buff;
35
struct sock_reuseport;
36

37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
/* ArgX, context and stack frame pointer register positions. Note,
 * Arg1, Arg2, Arg3, etc are used as argument mappings of function
 * calls in BPF_CALL instruction.
 */
#define BPF_REG_ARG1	BPF_REG_1
#define BPF_REG_ARG2	BPF_REG_2
#define BPF_REG_ARG3	BPF_REG_3
#define BPF_REG_ARG4	BPF_REG_4
#define BPF_REG_ARG5	BPF_REG_5
#define BPF_REG_CTX	BPF_REG_6
#define BPF_REG_FP	BPF_REG_10

/* Additional register mappings for converted user programs. */
#define BPF_REG_A	BPF_REG_0
#define BPF_REG_X	BPF_REG_7
52 53 54
#define BPF_REG_TMP	BPF_REG_2	/* scratch reg */
#define BPF_REG_D	BPF_REG_8	/* data, callee-saved */
#define BPF_REG_H	BPF_REG_9	/* hlen, callee-saved */
55

56
/* Kernel hidden auxiliary/helper register. */
57
#define BPF_REG_AX		MAX_BPF_REG
58 59
#define MAX_BPF_EXT_REG		(MAX_BPF_REG + 1)
#define MAX_BPF_JIT_REG		MAX_BPF_EXT_REG
60

61 62 63
/* unused opcode to mark special call to bpf_tail_call() helper */
#define BPF_TAIL_CALL	0xf0

64 65 66
/* unused opcode to mark call to interpreter with arguments */
#define BPF_CALL_ARGS	0xe0

67 68 69 70 71 72
/* As per nm, we expose JITed images as text (code) section for
 * kallsyms. That way, tools like perf can find it to match
 * addresses.
 */
#define BPF_SYM_ELF_TYPE	't'

73 74 75
/* BPF program can access up to 512 bytes of stack space. */
#define MAX_BPF_STACK	512

76 77
/* Helper macros for filter block array initializers. */

78
/* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
79

80
#define BPF_ALU64_REG(OP, DST, SRC)				\
81
	((struct bpf_insn) {					\
82
		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_X,	\
83 84
		.dst_reg = DST,					\
		.src_reg = SRC,					\
85 86 87
		.off   = 0,					\
		.imm   = 0 })

88
#define BPF_ALU32_REG(OP, DST, SRC)				\
89
	((struct bpf_insn) {					\
90
		.code  = BPF_ALU | BPF_OP(OP) | BPF_X,		\
91 92
		.dst_reg = DST,					\
		.src_reg = SRC,					\
93 94 95
		.off   = 0,					\
		.imm   = 0 })

96
/* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */
97

98
#define BPF_ALU64_IMM(OP, DST, IMM)				\
99
	((struct bpf_insn) {					\
100
		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_K,	\
101 102
		.dst_reg = DST,					\
		.src_reg = 0,					\
103 104 105
		.off   = 0,					\
		.imm   = IMM })

106
#define BPF_ALU32_IMM(OP, DST, IMM)				\
107
	((struct bpf_insn) {					\
108
		.code  = BPF_ALU | BPF_OP(OP) | BPF_K,		\
109 110
		.dst_reg = DST,					\
		.src_reg = 0,					\
111 112 113 114 115
		.off   = 0,					\
		.imm   = IMM })

/* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */

116
#define BPF_ENDIAN(TYPE, DST, LEN)				\
117
	((struct bpf_insn) {					\
118
		.code  = BPF_ALU | BPF_END | BPF_SRC(TYPE),	\
119 120
		.dst_reg = DST,					\
		.src_reg = 0,					\
121 122 123
		.off   = 0,					\
		.imm   = LEN })

124
/* Short form of mov, dst_reg = src_reg */
125

126
#define BPF_MOV64_REG(DST, SRC)					\
127
	((struct bpf_insn) {					\
128
		.code  = BPF_ALU64 | BPF_MOV | BPF_X,		\
129 130
		.dst_reg = DST,					\
		.src_reg = SRC,					\
131 132 133
		.off   = 0,					\
		.imm   = 0 })

134
#define BPF_MOV32_REG(DST, SRC)					\
135
	((struct bpf_insn) {					\
136
		.code  = BPF_ALU | BPF_MOV | BPF_X,		\
137 138
		.dst_reg = DST,					\
		.src_reg = SRC,					\
139 140 141
		.off   = 0,					\
		.imm   = 0 })

142
/* Short form of mov, dst_reg = imm32 */
143

144
#define BPF_MOV64_IMM(DST, IMM)					\
145
	((struct bpf_insn) {					\
146
		.code  = BPF_ALU64 | BPF_MOV | BPF_K,		\
147 148
		.dst_reg = DST,					\
		.src_reg = 0,					\
149 150 151
		.off   = 0,					\
		.imm   = IMM })

152
#define BPF_MOV32_IMM(DST, IMM)					\
153
	((struct bpf_insn) {					\
154
		.code  = BPF_ALU | BPF_MOV | BPF_K,		\
155 156
		.dst_reg = DST,					\
		.src_reg = 0,					\
157 158 159
		.off   = 0,					\
		.imm   = IMM })

160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
/* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
#define BPF_LD_IMM64(DST, IMM)					\
	BPF_LD_IMM64_RAW(DST, 0, IMM)

#define BPF_LD_IMM64_RAW(DST, SRC, IMM)				\
	((struct bpf_insn) {					\
		.code  = BPF_LD | BPF_DW | BPF_IMM,		\
		.dst_reg = DST,					\
		.src_reg = SRC,					\
		.off   = 0,					\
		.imm   = (__u32) (IMM) }),			\
	((struct bpf_insn) {					\
		.code  = 0, /* zero is reserved opcode */	\
		.dst_reg = 0,					\
		.src_reg = 0,					\
		.off   = 0,					\
		.imm   = ((__u64) (IMM)) >> 32 })

178 179 180 181
/* pseudo BPF_LD_IMM64 insn used to refer to process-local map_fd */
#define BPF_LD_MAP_FD(DST, MAP_FD)				\
	BPF_LD_IMM64_RAW(DST, BPF_PSEUDO_MAP_FD, MAP_FD)

182
/* Short form of mov based on type, BPF_X: dst_reg = src_reg, BPF_K: dst_reg = imm32 */
183

184
#define BPF_MOV64_RAW(TYPE, DST, SRC, IMM)			\
185
	((struct bpf_insn) {					\
186
		.code  = BPF_ALU64 | BPF_MOV | BPF_SRC(TYPE),	\
187 188
		.dst_reg = DST,					\
		.src_reg = SRC,					\
189 190 191
		.off   = 0,					\
		.imm   = IMM })

192
#define BPF_MOV32_RAW(TYPE, DST, SRC, IMM)			\
193
	((struct bpf_insn) {					\
194
		.code  = BPF_ALU | BPF_MOV | BPF_SRC(TYPE),	\
195 196
		.dst_reg = DST,					\
		.src_reg = SRC,					\
197 198 199
		.off   = 0,					\
		.imm   = IMM })

200
/* Direct packet access, R0 = *(uint *) (skb->data + imm32) */
201

202
#define BPF_LD_ABS(SIZE, IMM)					\
203
	((struct bpf_insn) {					\
204
		.code  = BPF_LD | BPF_SIZE(SIZE) | BPF_ABS,	\
205 206
		.dst_reg = 0,					\
		.src_reg = 0,					\
207
		.off   = 0,					\
208
		.imm   = IMM })
209

210
/* Indirect packet access, R0 = *(uint *) (skb->data + src_reg + imm32) */
211

212
#define BPF_LD_IND(SIZE, SRC, IMM)				\
213
	((struct bpf_insn) {					\
214
		.code  = BPF_LD | BPF_SIZE(SIZE) | BPF_IND,	\
215 216
		.dst_reg = 0,					\
		.src_reg = SRC,					\
217
		.off   = 0,					\
218
		.imm   = IMM })
219

220
/* Memory load, dst_reg = *(uint *) (src_reg + off16) */
221

222
#define BPF_LDX_MEM(SIZE, DST, SRC, OFF)			\
223
	((struct bpf_insn) {					\
224
		.code  = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM,	\
225 226
		.dst_reg = DST,					\
		.src_reg = SRC,					\
227 228 229
		.off   = OFF,					\
		.imm   = 0 })

230 231 232
/* Memory store, *(uint *) (dst_reg + off16) = src_reg */

#define BPF_STX_MEM(SIZE, DST, SRC, OFF)			\
233
	((struct bpf_insn) {					\
234
		.code  = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM,	\
235 236
		.dst_reg = DST,					\
		.src_reg = SRC,					\
237 238 239
		.off   = OFF,					\
		.imm   = 0 })

240 241 242 243 244 245 246 247 248 249
/* Atomic memory add, *(uint *)(dst_reg + off16) += src_reg */

#define BPF_STX_XADD(SIZE, DST, SRC, OFF)			\
	((struct bpf_insn) {					\
		.code  = BPF_STX | BPF_SIZE(SIZE) | BPF_XADD,	\
		.dst_reg = DST,					\
		.src_reg = SRC,					\
		.off   = OFF,					\
		.imm   = 0 })

250 251 252
/* Memory store, *(uint *) (dst_reg + off16) = imm32 */

#define BPF_ST_MEM(SIZE, DST, OFF, IMM)				\
253
	((struct bpf_insn) {					\
254 255 256 257 258 259 260
		.code  = BPF_ST | BPF_SIZE(SIZE) | BPF_MEM,	\
		.dst_reg = DST,					\
		.src_reg = 0,					\
		.off   = OFF,					\
		.imm   = IMM })

/* Conditional jumps against registers, if (dst_reg 'op' src_reg) goto pc + off16 */
261

262
#define BPF_JMP_REG(OP, DST, SRC, OFF)				\
263
	((struct bpf_insn) {					\
264
		.code  = BPF_JMP | BPF_OP(OP) | BPF_X,		\
265 266
		.dst_reg = DST,					\
		.src_reg = SRC,					\
267 268 269
		.off   = OFF,					\
		.imm   = 0 })

270
/* Conditional jumps against immediates, if (dst_reg 'op' imm32) goto pc + off16 */
271

272
#define BPF_JMP_IMM(OP, DST, IMM, OFF)				\
273
	((struct bpf_insn) {					\
274
		.code  = BPF_JMP | BPF_OP(OP) | BPF_K,		\
275 276
		.dst_reg = DST,					\
		.src_reg = 0,					\
277 278 279
		.off   = OFF,					\
		.imm   = IMM })

280 281 282 283 284 285 286 287 288 289
/* Unconditional jumps, goto pc + off16 */

#define BPF_JMP_A(OFF)						\
	((struct bpf_insn) {					\
		.code  = BPF_JMP | BPF_JA,			\
		.dst_reg = 0,					\
		.src_reg = 0,					\
		.off   = OFF,					\
		.imm   = 0 })

290 291 292 293 294 295 296 297 298 299
/* Relative call */

#define BPF_CALL_REL(TGT)					\
	((struct bpf_insn) {					\
		.code  = BPF_JMP | BPF_CALL,			\
		.dst_reg = 0,					\
		.src_reg = BPF_PSEUDO_CALL,			\
		.off   = 0,					\
		.imm   = TGT })

300 301
/* Function call */

302 303 304
#define BPF_CAST_CALL(x)					\
		((u64 (*)(u64, u64, u64, u64, u64))(x))

305
#define BPF_EMIT_CALL(FUNC)					\
306
	((struct bpf_insn) {					\
307
		.code  = BPF_JMP | BPF_CALL,			\
308 309
		.dst_reg = 0,					\
		.src_reg = 0,					\
310 311 312 313 314
		.off   = 0,					\
		.imm   = ((FUNC) - __bpf_call_base) })

/* Raw code statement block */

315
#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM)			\
316
	((struct bpf_insn) {					\
317
		.code  = CODE,					\
318 319
		.dst_reg = DST,					\
		.src_reg = SRC,					\
320 321 322 323 324 325
		.off   = OFF,					\
		.imm   = IMM })

/* Program exit */

#define BPF_EXIT_INSN()						\
326
	((struct bpf_insn) {					\
327
		.code  = BPF_JMP | BPF_EXIT,			\
328 329
		.dst_reg = 0,					\
		.src_reg = 0,					\
330 331 332
		.off   = 0,					\
		.imm   = 0 })

333 334 335 336 337 338 339 340
/* Internal classic blocks for direct assignment */

#define __BPF_STMT(CODE, K)					\
	((struct sock_filter) BPF_STMT(CODE, K))

#define __BPF_JUMP(CODE, K, JT, JF)				\
	((struct sock_filter) BPF_JUMP(CODE, K, JT, JF))

341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
#define bytes_to_bpf_size(bytes)				\
({								\
	int bpf_size = -EINVAL;					\
								\
	if (bytes == sizeof(u8))				\
		bpf_size = BPF_B;				\
	else if (bytes == sizeof(u16))				\
		bpf_size = BPF_H;				\
	else if (bytes == sizeof(u32))				\
		bpf_size = BPF_W;				\
	else if (bytes == sizeof(u64))				\
		bpf_size = BPF_DW;				\
								\
	bpf_size;						\
})
356

357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
#define bpf_size_to_bytes(bpf_size)				\
({								\
	int bytes = -EINVAL;					\
								\
	if (bpf_size == BPF_B)					\
		bytes = sizeof(u8);				\
	else if (bpf_size == BPF_H)				\
		bytes = sizeof(u16);				\
	else if (bpf_size == BPF_W)				\
		bytes = sizeof(u32);				\
	else if (bpf_size == BPF_DW)				\
		bytes = sizeof(u64);				\
								\
	bytes;							\
})

373 374 375 376 377 378 379 380 381 382 383 384 385 386
#define BPF_SIZEOF(type)					\
	({							\
		const int __size = bytes_to_bpf_size(sizeof(type)); \
		BUILD_BUG_ON(__size < 0);			\
		__size;						\
	})

#define BPF_FIELD_SIZEOF(type, field)				\
	({							\
		const int __size = bytes_to_bpf_size(FIELD_SIZEOF(type, field)); \
		BUILD_BUG_ON(__size < 0);			\
		__size;						\
	})

387 388
#define BPF_LDST_BYTES(insn)					\
	({							\
389
		const int __size = bpf_size_to_bytes(BPF_SIZE((insn)->code)); \
390 391 392 393
		WARN_ON(__size < 0);				\
		__size;						\
	})

394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
#define __BPF_MAP_0(m, v, ...) v
#define __BPF_MAP_1(m, v, t, a, ...) m(t, a)
#define __BPF_MAP_2(m, v, t, a, ...) m(t, a), __BPF_MAP_1(m, v, __VA_ARGS__)
#define __BPF_MAP_3(m, v, t, a, ...) m(t, a), __BPF_MAP_2(m, v, __VA_ARGS__)
#define __BPF_MAP_4(m, v, t, a, ...) m(t, a), __BPF_MAP_3(m, v, __VA_ARGS__)
#define __BPF_MAP_5(m, v, t, a, ...) m(t, a), __BPF_MAP_4(m, v, __VA_ARGS__)

#define __BPF_REG_0(...) __BPF_PAD(5)
#define __BPF_REG_1(...) __BPF_MAP(1, __VA_ARGS__), __BPF_PAD(4)
#define __BPF_REG_2(...) __BPF_MAP(2, __VA_ARGS__), __BPF_PAD(3)
#define __BPF_REG_3(...) __BPF_MAP(3, __VA_ARGS__), __BPF_PAD(2)
#define __BPF_REG_4(...) __BPF_MAP(4, __VA_ARGS__), __BPF_PAD(1)
#define __BPF_REG_5(...) __BPF_MAP(5, __VA_ARGS__)

#define __BPF_MAP(n, ...) __BPF_MAP_##n(__VA_ARGS__)
#define __BPF_REG(n, ...) __BPF_REG_##n(__VA_ARGS__)

#define __BPF_CAST(t, a)						       \
	(__force t)							       \
	(__force							       \
	 typeof(__builtin_choose_expr(sizeof(t) == sizeof(unsigned long),      \
				      (unsigned long)0, (t)0))) a
#define __BPF_V void
#define __BPF_N

#define __BPF_DECL_ARGS(t, a) t   a
#define __BPF_DECL_REGS(t, a) u64 a

#define __BPF_PAD(n)							       \
	__BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2,       \
		  u64, __ur_3, u64, __ur_4, u64, __ur_5)

#define BPF_CALL_x(x, name, ...)					       \
	static __always_inline						       \
	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__));   \
	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__));	       \
	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__))	       \
	{								       \
		return ____##name(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
	}								       \
	static __always_inline						       \
	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))

#define BPF_CALL_0(name, ...)	BPF_CALL_x(0, name, __VA_ARGS__)
#define BPF_CALL_1(name, ...)	BPF_CALL_x(1, name, __VA_ARGS__)
#define BPF_CALL_2(name, ...)	BPF_CALL_x(2, name, __VA_ARGS__)
#define BPF_CALL_3(name, ...)	BPF_CALL_x(3, name, __VA_ARGS__)
#define BPF_CALL_4(name, ...)	BPF_CALL_x(4, name, __VA_ARGS__)
#define BPF_CALL_5(name, ...)	BPF_CALL_x(5, name, __VA_ARGS__)

444 445 446 447
#define bpf_ctx_range(TYPE, MEMBER)						\
	offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
#define bpf_ctx_range_till(TYPE, MEMBER1, MEMBER2)				\
	offsetof(TYPE, MEMBER1) ... offsetofend(TYPE, MEMBER2) - 1
448 449 450 451 452 453 454
#if BITS_PER_LONG == 64
# define bpf_ctx_range_ptr(TYPE, MEMBER)					\
	offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
#else
# define bpf_ctx_range_ptr(TYPE, MEMBER)					\
	offsetof(TYPE, MEMBER) ... offsetof(TYPE, MEMBER) + 8 - 1
#endif /* BITS_PER_LONG == 64 */
455 456 457 458 459 460 461 462

#define bpf_target_off(TYPE, MEMBER, SIZE, PTR_SIZE)				\
	({									\
		BUILD_BUG_ON(FIELD_SIZEOF(TYPE, MEMBER) != (SIZE));		\
		*(PTR_SIZE) = (SIZE);						\
		offsetof(TYPE, MEMBER);						\
	})

463 464
#ifdef CONFIG_COMPAT
/* A struct sock_filter is architecture independent. */
465 466
struct compat_sock_fprog {
	u16		len;
467
	compat_uptr_t	filter;	/* struct sock_filter * */
468 469 470
};
#endif

471 472 473 474 475
struct sock_fprog_kern {
	u16			len;
	struct sock_filter	*filter;
};

476
struct bpf_binary_header {
477
	u32 pages;
478 479
	/* Some arches need word alignment for their instructions */
	u8 image[] __aligned(4);
480 481
};

482
struct bpf_prog {
483
	u16			pages;		/* Number of allocated pages */
484
	u16			jited:1,	/* Is our filter JIT'ed? */
485
				jit_requested:1,/* archs need to JIT the prog */
486
				undo_set_mem:1,	/* Passed set_memory_ro() checkpoint */
487
				gpl_compatible:1, /* Is filter GPL compatible? */
488
				cb_access:1,	/* Is control block accessed? */
489
				dst_needed:1,	/* Do we need dst entry? */
490 491
				blinded:1,	/* Was blinded */
				is_func:1,	/* program is a bpf function */
492 493
				kprobe_override:1, /* Do we override a kprobe? */
				has_callchain_buf:1; /* callchain buffer allocated? */
494
	enum bpf_prog_type	type;		/* Type of BPF program */
495
	enum bpf_attach_type	expected_attach_type; /* For some prog types */
496
	u32			len;		/* Number of filter blocks */
497
	u32			jited_len;	/* Size of jited insns in bytes */
498
	u8			tag[BPF_TAG_SIZE];
499
	struct bpf_prog_aux	*aux;		/* Auxiliary fields */
500
	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */
501 502
	unsigned int		(*bpf_func)(const void *ctx,
					    const struct bpf_insn *insn);
503
	/* Instructions for interpreter */
504
	union {
505
		struct sock_filter	insns[0];
506
		struct bpf_insn		insnsi[0];
507
	};
508 509
};

510
struct sk_filter {
511
	refcount_t	refcnt;
512 513 514 515
	struct rcu_head	rcu;
	struct bpf_prog	*prog;
};

516
#define BPF_PROG_RUN(filter, ctx)  (*(filter)->bpf_func)(ctx, (filter)->insnsi)
517

518 519
#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN

520 521
struct bpf_skb_data_end {
	struct qdisc_skb_cb qdisc_cb;
522
	void *data_meta;
523 524 525
	void *data_end;
};

526 527 528 529 530
struct bpf_redirect_info {
	u32 ifindex;
	u32 flags;
	struct bpf_map *map;
	struct bpf_map *map_to_flush;
531
	u32 kern_flags;
532 533 534 535
};

DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);

536 537 538
/* flags for bpf_redirect_info kern_flags */
#define BPF_RI_F_RF_NO_DIRECT	BIT(0)	/* no napi_direct on return_frame */

539 540 541 542 543
/* Compute the linear packet data range [data, data_end) which
 * will be accessed by various program types (cls_bpf, act_bpf,
 * lwt, ...). Subsystems allowing direct data access must (!)
 * ensure that cb[] area can be written to when BPF program is
 * invoked (otherwise cb[] save/restore is necessary).
544
 */
545
static inline void bpf_compute_data_pointers(struct sk_buff *skb)
546 547 548 549
{
	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;

	BUILD_BUG_ON(sizeof(*cb) > FIELD_SIZEOF(struct sk_buff, cb));
550 551
	cb->data_meta = skb->data - skb_metadata_len(skb);
	cb->data_end  = skb->data + skb_headlen(skb);
552 553
}

554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
/* Similar to bpf_compute_data_pointers(), except that save orginal
 * data in cb->data and cb->meta_data for restore.
 */
static inline void bpf_compute_and_save_data_end(
	struct sk_buff *skb, void **saved_data_end)
{
	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;

	*saved_data_end = cb->data_end;
	cb->data_end  = skb->data + skb_headlen(skb);
}

/* Restore data saved by bpf_compute_data_pointers(). */
static inline void bpf_restore_data_end(
	struct sk_buff *skb, void *saved_data_end)
{
	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;

	cb->data_end = saved_data_end;
}

575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
static inline u8 *bpf_skb_cb(struct sk_buff *skb)
{
	/* eBPF programs may read/write skb->cb[] area to transfer meta
	 * data between tail calls. Since this also needs to work with
	 * tc, that scratch memory is mapped to qdisc_skb_cb's data area.
	 *
	 * In some socket filter cases, the cb unfortunately needs to be
	 * saved/restored so that protocol specific skb->cb[] data won't
	 * be lost. In any case, due to unpriviledged eBPF programs
	 * attached to sockets, we need to clear the bpf_skb_cb() area
	 * to not leak previous contents to user space.
	 */
	BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != BPF_SKB_CB_LEN);
	BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) !=
		     FIELD_SIZEOF(struct qdisc_skb_cb, data));

	return qdisc_skb_cb(skb)->data;
}

594 595 596
static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog,
				       struct sk_buff *skb)
{
597 598
	u8 *cb_data = bpf_skb_cb(skb);
	u8 cb_saved[BPF_SKB_CB_LEN];
599 600 601
	u32 res;

	if (unlikely(prog->cb_access)) {
602 603
		memcpy(cb_saved, cb_data, sizeof(cb_saved));
		memset(cb_data, 0, sizeof(cb_saved));
604 605 606 607 608
	}

	res = BPF_PROG_RUN(prog, skb);

	if (unlikely(prog->cb_access))
609
		memcpy(cb_data, cb_saved, sizeof(cb_saved));
610 611 612 613 614 615 616

	return res;
}

static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog,
					struct sk_buff *skb)
{
617
	u8 *cb_data = bpf_skb_cb(skb);
618 619

	if (unlikely(prog->cb_access))
620 621
		memset(cb_data, 0, BPF_SKB_CB_LEN);

622 623 624
	return BPF_PROG_RUN(prog, skb);
}

625 626
static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
					    struct xdp_buff *xdp)
627
{
628 629 630 631 632 633 634
	/* Caller needs to hold rcu_read_lock() (!), otherwise program
	 * can be released while still running, or map elements could be
	 * freed early while still having concurrent users. XDP fastpath
	 * already takes rcu_read_lock() when fetching the program, so
	 * it's not necessary here anymore.
	 */
	return BPF_PROG_RUN(prog, xdp);
635 636
}

637 638 639 640 641
static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
{
	return prog->len * sizeof(struct bpf_insn);
}

642
static inline u32 bpf_prog_tag_scratch_size(const struct bpf_prog *prog)
643 644 645 646 647
{
	return round_up(bpf_prog_insn_size(prog) +
			sizeof(__be64) + 1, SHA_MESSAGE_BYTES);
}

648
static inline unsigned int bpf_prog_size(unsigned int proglen)
649
{
650 651
	return max(sizeof(struct bpf_prog),
		   offsetof(struct bpf_prog, insns[proglen]));
652 653
}

654 655 656 657 658 659 660 661 662 663
static inline bool bpf_prog_was_classic(const struct bpf_prog *prog)
{
	/* When classic BPF programs have been loaded and the arch
	 * does not have a classic BPF JIT (anymore), they have been
	 * converted via bpf_migrate_filter() to eBPF and thus always
	 * have an unspec program type.
	 */
	return prog->type == BPF_PROG_TYPE_UNSPEC;
}

664 665 666 667 668 669 670 671 672 673 674 675 676
static inline u32 bpf_ctx_off_adjust_machine(u32 size)
{
	const u32 size_machine = sizeof(unsigned long);

	if (size > size_machine && size % size_machine == 0)
		size = size_machine;

	return size;
}

static inline bool
bpf_ctx_narrow_access_ok(u32 off, u32 size, u32 size_default)
{
677
	return size <= size_default && (size & (size - 1)) == 0;
678 679
}

680
#define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0]))
681

682 683
static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
{
684 685
	fp->undo_set_mem = 1;
	set_memory_ro((unsigned long)fp, fp->pages);
686 687 688 689
}

static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
{
690 691
	if (fp->undo_set_mem)
		set_memory_rw((unsigned long)fp, fp->pages);
692
}
693

694 695
static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
{
696
	set_memory_ro((unsigned long)hdr, hdr->pages);
697 698
}

699 700
static inline void bpf_jit_binary_unlock_ro(struct bpf_binary_header *hdr)
{
701
	set_memory_rw((unsigned long)hdr, hdr->pages);
702
}
703

704 705 706 707 708 709 710 711 712
static inline struct bpf_binary_header *
bpf_jit_binary_hdr(const struct bpf_prog *fp)
{
	unsigned long real_start = (unsigned long)fp->bpf_func;
	unsigned long addr = real_start & PAGE_MASK;

	return (void *)addr;
}

713 714 715 716 717
int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
{
	return sk_filter_trim_cap(sk, skb, 1);
}
718

719
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
720
void bpf_prog_free(struct bpf_prog *fp);
721

722 723
bool bpf_opcode_in_insntable(u8 code);

724 725 726 727 728 729 730
void bpf_prog_free_linfo(struct bpf_prog *prog);
void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
			       const u32 *insn_to_jit_off);
int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
void bpf_prog_free_jited_linfo(struct bpf_prog *prog);
void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);

731 732 733 734 735 736 737 738 739 740 741
struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
				  gfp_t gfp_extra_flags);
void __bpf_prog_free(struct bpf_prog *fp);

static inline void bpf_prog_unlock_free(struct bpf_prog *fp)
{
	bpf_prog_unlock_ro(fp);
	__bpf_prog_free(fp);
}

742 743 744
typedef int (*bpf_aux_classic_check_t)(struct sock_filter *filter,
				       unsigned int flen);

745
int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog);
746
int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
747
			      bpf_aux_classic_check_t trans, bool save_orig);
748
void bpf_prog_destroy(struct bpf_prog *fp);
749

750
int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk);
751
int sk_attach_bpf(u32 ufd, struct sock *sk);
752 753
int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk);
int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk);
754
void sk_reuseport_prog_free(struct bpf_prog *prog);
755 756 757 758
int sk_detach_filter(struct sock *sk);
int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,
		  unsigned int len);

759
bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
760
void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
761

762
u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
763 764 765
#define __bpf_call_base_args \
	((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
	 __bpf_call_base)
766 767

struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
768
void bpf_jit_compile(struct bpf_prog *prog);
769
bool bpf_helper_changes_pkt_data(void *func);
770

771 772 773 774 775 776 777 778
static inline bool bpf_dump_raw_ok(void)
{
	/* Reconstruction of call-sites is dependent on kallsyms,
	 * thus make dump the same restriction.
	 */
	return kallsyms_show_value() == 1;
}

779 780
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
				       const struct bpf_insn *patch, u32 len);
781

782 783
void bpf_clear_redirect_map(struct bpf_map *map);

784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
static inline bool xdp_return_frame_no_direct(void)
{
	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

	return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT;
}

static inline void xdp_set_return_frame_no_direct(void)
{
	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

	ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT;
}

static inline void xdp_clear_return_frame_no_direct(void)
{
	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

	ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT;
}

805 806
static inline int xdp_ok_fwd_dev(const struct net_device *fwd,
				 unsigned int pktlen)
807 808 809 810 811 812 813
{
	unsigned int len;

	if (unlikely(!(fwd->flags & IFF_UP)))
		return -ENETDOWN;

	len = fwd->mtu + fwd->hard_header_len + VLAN_HLEN;
814
	if (pktlen > len)
815 816 817 818 819
		return -EMSGSIZE;

	return 0;
}

820 821 822 823
/* The pair of xdp_do_redirect and xdp_do_flush_map MUST be called in the
 * same cpu context. Further for best results no more than a single map
 * for the do_redirect/do_flush pair should be used. This limitation is
 * because we only track one map and force a flush when the map changes.
824
 * This does not appear to be a real limitation for existing software.
825
 */
826
int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
827
			    struct xdp_buff *xdp, struct bpf_prog *prog);
828 829 830
int xdp_do_redirect(struct net_device *dev,
		    struct xdp_buff *xdp,
		    struct bpf_prog *prog);
831
void xdp_do_flush_map(void);
832

833
void bpf_warn_invalid_xdp_action(u32 act);
834

835 836 837 838 839 840 841 842 843 844 845 846 847 848
#ifdef CONFIG_INET
struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
				  struct bpf_prog *prog, struct sk_buff *skb,
				  u32 hash);
#else
static inline struct sock *
bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
		     struct bpf_prog *prog, struct sk_buff *skb,
		     u32 hash)
{
	return NULL;
}
#endif

849
#ifdef CONFIG_BPF_JIT
850
extern int bpf_jit_enable;
851
extern int bpf_jit_harden;
852
extern int bpf_jit_kallsyms;
853
extern long bpf_jit_limit;
854

855 856 857 858 859 860 861 862 863 864
typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);

struct bpf_binary_header *
bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
		     unsigned int alignment,
		     bpf_jit_fill_hole_t bpf_fill_ill_insns);
void bpf_jit_binary_free(struct bpf_binary_header *hdr);

void bpf_jit_free(struct bpf_prog *fp);

865 866 867 868
int bpf_jit_get_func_addr(const struct bpf_prog *prog,
			  const struct bpf_insn *insn, bool extra_pass,
			  u64 *func_addr, bool *func_addr_fixed);

869 870 871
struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);

872 873 874
static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
				u32 pass, void *image)
{
875 876 877
	pr_err("flen=%u proglen=%u pass=%u image=%pK from=%s pid=%d\n", flen,
	       proglen, pass, image, current->comm, task_pid_nr(current));

878 879 880 881
	if (image)
		print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_OFFSET,
			       16, 1, image, proglen, false);
}
882 883 884 885 886 887 888 889 890 891

static inline bool bpf_jit_is_ebpf(void)
{
# ifdef CONFIG_HAVE_EBPF_JIT
	return true;
# else
	return false;
# endif
}

892 893 894 895 896
static inline bool ebpf_jit_enabled(void)
{
	return bpf_jit_enable && bpf_jit_is_ebpf();
}

897 898 899 900 901
static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp)
{
	return fp->jited && bpf_jit_is_ebpf();
}

902
static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog)
903 904 905 906 907 908 909
{
	/* These are the prerequisites, should someone ever have the
	 * idea to call blinding outside of them, we make sure to
	 * bail out.
	 */
	if (!bpf_jit_is_ebpf())
		return false;
910
	if (!prog->jit_requested)
911 912 913 914 915 916 917 918
		return false;
	if (!bpf_jit_harden)
		return false;
	if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN))
		return false;

	return true;
}
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956

static inline bool bpf_jit_kallsyms_enabled(void)
{
	/* There are a couple of corner cases where kallsyms should
	 * not be enabled f.e. on hardening.
	 */
	if (bpf_jit_harden)
		return false;
	if (!bpf_jit_kallsyms)
		return false;
	if (bpf_jit_kallsyms == 1)
		return true;

	return false;
}

const char *__bpf_address_lookup(unsigned long addr, unsigned long *size,
				 unsigned long *off, char *sym);
bool is_bpf_text_address(unsigned long addr);
int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
		    char *sym);

static inline const char *
bpf_address_lookup(unsigned long addr, unsigned long *size,
		   unsigned long *off, char **modname, char *sym)
{
	const char *ret = __bpf_address_lookup(addr, size, off, sym);

	if (ret && modname)
		*modname = NULL;
	return ret;
}

void bpf_prog_kallsyms_add(struct bpf_prog *fp);
void bpf_prog_kallsyms_del(struct bpf_prog *fp);

#else /* CONFIG_BPF_JIT */

957 958 959 960 961
static inline bool ebpf_jit_enabled(void)
{
	return false;
}

962 963 964 965 966
static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp)
{
	return false;
}

967 968 969 970
static inline void bpf_jit_free(struct bpf_prog *fp)
{
	bpf_prog_unlock_free(fp);
}
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008

static inline bool bpf_jit_kallsyms_enabled(void)
{
	return false;
}

static inline const char *
__bpf_address_lookup(unsigned long addr, unsigned long *size,
		     unsigned long *off, char *sym)
{
	return NULL;
}

static inline bool is_bpf_text_address(unsigned long addr)
{
	return false;
}

static inline int bpf_get_kallsym(unsigned int symnum, unsigned long *value,
				  char *type, char *sym)
{
	return -ERANGE;
}

static inline const char *
bpf_address_lookup(unsigned long addr, unsigned long *size,
		   unsigned long *off, char **modname, char *sym)
{
	return NULL;
}

static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp)
{
}

static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp)
{
}
1009 1010
#endif /* CONFIG_BPF_JIT */

1011 1012 1013
void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp);
void bpf_prog_kallsyms_del_all(struct bpf_prog *fp);

1014 1015
#define BPF_ANC		BIT(15)

1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
static inline bool bpf_needs_clear_a(const struct sock_filter *first)
{
	switch (first->code) {
	case BPF_RET | BPF_K:
	case BPF_LD | BPF_W | BPF_LEN:
		return false;

	case BPF_LD | BPF_W | BPF_ABS:
	case BPF_LD | BPF_H | BPF_ABS:
	case BPF_LD | BPF_B | BPF_ABS:
		if (first->k == SKF_AD_OFF + SKF_AD_ALU_XOR_X)
			return true;
		return false;

	default:
		return true;
	}
}

1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
static inline u16 bpf_anc_helper(const struct sock_filter *ftest)
{
	BUG_ON(ftest->code & BPF_ANC);

	switch (ftest->code) {
	case BPF_LD | BPF_W | BPF_ABS:
	case BPF_LD | BPF_H | BPF_ABS:
	case BPF_LD | BPF_B | BPF_ABS:
#define BPF_ANCILLARY(CODE)	case SKF_AD_OFF + SKF_AD_##CODE:	\
				return BPF_ANC | SKF_AD_##CODE
		switch (ftest->k) {
		BPF_ANCILLARY(PROTOCOL);
		BPF_ANCILLARY(PKTTYPE);
		BPF_ANCILLARY(IFINDEX);
		BPF_ANCILLARY(NLATTR);
		BPF_ANCILLARY(NLATTR_NEST);
		BPF_ANCILLARY(MARK);
		BPF_ANCILLARY(QUEUE);
		BPF_ANCILLARY(HATYPE);
		BPF_ANCILLARY(RXHASH);
		BPF_ANCILLARY(CPU);
		BPF_ANCILLARY(ALU_XOR_X);
		BPF_ANCILLARY(VLAN_TAG);
		BPF_ANCILLARY(VLAN_TAG_PRESENT);
		BPF_ANCILLARY(PAY_OFFSET);
		BPF_ANCILLARY(RANDOM);
1061
		BPF_ANCILLARY(VLAN_TPID);
1062 1063 1064 1065 1066 1067 1068
		}
		/* Fallthrough. */
	default:
		return ftest->code;
	}
}

1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
					   int k, unsigned int size);

static inline void *bpf_load_pointer(const struct sk_buff *skb, int k,
				     unsigned int size, void *buffer)
{
	if (k >= 0)
		return skb_header_pointer(skb, k, size, buffer);

	return bpf_internal_load_pointer_neg_helper(skb, k, size);
}

1081 1082
static inline int bpf_tell_extensions(void)
{
1083
	return SKF_AD_MAX;
1084 1085
}

1086 1087 1088 1089 1090 1091 1092 1093
struct bpf_sock_addr_kern {
	struct sock *sk;
	struct sockaddr *uaddr;
	/* Temporary "register" to make indirect stores to nested structures
	 * defined above. We need three registers to make such a store, but
	 * only two (src and dst) are available at convert_ctx_access time
	 */
	u64 tmp_reg;
1094
	void *t_ctx;	/* Attach type specific context. */
1095 1096
};

1097 1098 1099 1100
struct bpf_sock_ops_kern {
	struct	sock *sk;
	u32	op;
	union {
1101
		u32 args[4];
1102 1103 1104
		u32 reply;
		u32 replylong[4];
	};
1105
	u32	is_fullsock;
1106 1107 1108 1109 1110 1111 1112 1113 1114
	u64	temp;			/* temp and everything after is not
					 * initialized to 0 before calling
					 * the BPF program. New fields that
					 * should be initialized to 0 should
					 * be inserted before temp.
					 * temp is scratch storage used by
					 * sock_ops_convert_ctx_access
					 * as temporary storage of a register.
					 */
1115 1116
};

L
Linus Torvalds 已提交
1117
#endif /* __LINUX_FILTER_H__ */
反馈
建议
客服 返回
顶部