filter.h 24.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 * Linux Socket Filter Data Structures
 */
#ifndef __LINUX_FILTER_H__
#define __LINUX_FILTER_H__

7 8
#include <stdarg.h>

A
Arun Sharma 已提交
9
#include <linux/atomic.h>
10
#include <linux/refcount.h>
11
#include <linux/compat.h>
12
#include <linux/skbuff.h>
13 14
#include <linux/linkage.h>
#include <linux/printk.h>
15
#include <linux/workqueue.h>
16
#include <linux/sched.h>
17
#include <linux/capability.h>
18
#include <linux/cryptohash.h>
19

20
#include <net/sch_generic.h>
21

22 23 24
#ifdef CONFIG_ARCH_HAS_SET_MEMORY
#include <asm/set_memory.h>
#endif
25 26

#include <uapi/linux/filter.h>
27
#include <uapi/linux/bpf.h>
28 29 30 31

struct sk_buff;
struct sock;
struct seccomp_data;
32
struct bpf_prog_aux;
33

34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
/* ArgX, context and stack frame pointer register positions. Note,
 * Arg1, Arg2, Arg3, etc are used as argument mappings of function
 * calls in BPF_CALL instruction.
 */
#define BPF_REG_ARG1	BPF_REG_1
#define BPF_REG_ARG2	BPF_REG_2
#define BPF_REG_ARG3	BPF_REG_3
#define BPF_REG_ARG4	BPF_REG_4
#define BPF_REG_ARG5	BPF_REG_5
#define BPF_REG_CTX	BPF_REG_6
#define BPF_REG_FP	BPF_REG_10

/* Additional register mappings for converted user programs. */
#define BPF_REG_A	BPF_REG_0
#define BPF_REG_X	BPF_REG_7
#define BPF_REG_TMP	BPF_REG_8
50

51 52 53 54 55 56 57 58 59
/* Kernel hidden auxiliary/helper register for hardening step.
 * Only used by eBPF JITs. It's nothing more than a temporary
 * register that JITs use internally, only that here it's part
 * of eBPF instructions that have been rewritten for blinding
 * constants. See JIT pre-step in bpf_jit_blind_constants().
 */
#define BPF_REG_AX		MAX_BPF_REG
#define MAX_BPF_JIT_REG		(MAX_BPF_REG + 1)

60 61 62
/* unused opcode to mark special call to bpf_tail_call() helper */
#define BPF_TAIL_CALL	0xf0

63 64 65 66 67 68
/* As per nm, we expose JITed images as text (code) section for
 * kallsyms. That way, tools like perf can find it to match
 * addresses.
 */
#define BPF_SYM_ELF_TYPE	't'

69 70 71
/* BPF program can access up to 512 bytes of stack space. */
#define MAX_BPF_STACK	512

72 73
/* Helper macros for filter block array initializers. */

74
/* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
75

76
#define BPF_ALU64_REG(OP, DST, SRC)				\
77
	((struct bpf_insn) {					\
78
		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_X,	\
79 80
		.dst_reg = DST,					\
		.src_reg = SRC,					\
81 82 83
		.off   = 0,					\
		.imm   = 0 })

84
#define BPF_ALU32_REG(OP, DST, SRC)				\
85
	((struct bpf_insn) {					\
86
		.code  = BPF_ALU | BPF_OP(OP) | BPF_X,		\
87 88
		.dst_reg = DST,					\
		.src_reg = SRC,					\
89 90 91
		.off   = 0,					\
		.imm   = 0 })

92
/* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */
93

94
#define BPF_ALU64_IMM(OP, DST, IMM)				\
95
	((struct bpf_insn) {					\
96
		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_K,	\
97 98
		.dst_reg = DST,					\
		.src_reg = 0,					\
99 100 101
		.off   = 0,					\
		.imm   = IMM })

102
#define BPF_ALU32_IMM(OP, DST, IMM)				\
103
	((struct bpf_insn) {					\
104
		.code  = BPF_ALU | BPF_OP(OP) | BPF_K,		\
105 106
		.dst_reg = DST,					\
		.src_reg = 0,					\
107 108 109 110 111
		.off   = 0,					\
		.imm   = IMM })

/* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */

112
#define BPF_ENDIAN(TYPE, DST, LEN)				\
113
	((struct bpf_insn) {					\
114
		.code  = BPF_ALU | BPF_END | BPF_SRC(TYPE),	\
115 116
		.dst_reg = DST,					\
		.src_reg = 0,					\
117 118 119
		.off   = 0,					\
		.imm   = LEN })

120
/* Short form of mov, dst_reg = src_reg */
121

122
#define BPF_MOV64_REG(DST, SRC)					\
123
	((struct bpf_insn) {					\
124
		.code  = BPF_ALU64 | BPF_MOV | BPF_X,		\
125 126
		.dst_reg = DST,					\
		.src_reg = SRC,					\
127 128 129
		.off   = 0,					\
		.imm   = 0 })

130
#define BPF_MOV32_REG(DST, SRC)					\
131
	((struct bpf_insn) {					\
132
		.code  = BPF_ALU | BPF_MOV | BPF_X,		\
133 134
		.dst_reg = DST,					\
		.src_reg = SRC,					\
135 136 137
		.off   = 0,					\
		.imm   = 0 })

138
/* Short form of mov, dst_reg = imm32 */
139

140
#define BPF_MOV64_IMM(DST, IMM)					\
141
	((struct bpf_insn) {					\
142
		.code  = BPF_ALU64 | BPF_MOV | BPF_K,		\
143 144
		.dst_reg = DST,					\
		.src_reg = 0,					\
145 146 147
		.off   = 0,					\
		.imm   = IMM })

148
#define BPF_MOV32_IMM(DST, IMM)					\
149
	((struct bpf_insn) {					\
150
		.code  = BPF_ALU | BPF_MOV | BPF_K,		\
151 152
		.dst_reg = DST,					\
		.src_reg = 0,					\
153 154 155
		.off   = 0,					\
		.imm   = IMM })

156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
/* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
#define BPF_LD_IMM64(DST, IMM)					\
	BPF_LD_IMM64_RAW(DST, 0, IMM)

#define BPF_LD_IMM64_RAW(DST, SRC, IMM)				\
	((struct bpf_insn) {					\
		.code  = BPF_LD | BPF_DW | BPF_IMM,		\
		.dst_reg = DST,					\
		.src_reg = SRC,					\
		.off   = 0,					\
		.imm   = (__u32) (IMM) }),			\
	((struct bpf_insn) {					\
		.code  = 0, /* zero is reserved opcode */	\
		.dst_reg = 0,					\
		.src_reg = 0,					\
		.off   = 0,					\
		.imm   = ((__u64) (IMM)) >> 32 })

174 175 176 177
/* pseudo BPF_LD_IMM64 insn used to refer to process-local map_fd */
#define BPF_LD_MAP_FD(DST, MAP_FD)				\
	BPF_LD_IMM64_RAW(DST, BPF_PSEUDO_MAP_FD, MAP_FD)

178
/* Short form of mov based on type, BPF_X: dst_reg = src_reg, BPF_K: dst_reg = imm32 */
179

180
#define BPF_MOV64_RAW(TYPE, DST, SRC, IMM)			\
181
	((struct bpf_insn) {					\
182
		.code  = BPF_ALU64 | BPF_MOV | BPF_SRC(TYPE),	\
183 184
		.dst_reg = DST,					\
		.src_reg = SRC,					\
185 186 187
		.off   = 0,					\
		.imm   = IMM })

188
#define BPF_MOV32_RAW(TYPE, DST, SRC, IMM)			\
189
	((struct bpf_insn) {					\
190
		.code  = BPF_ALU | BPF_MOV | BPF_SRC(TYPE),	\
191 192
		.dst_reg = DST,					\
		.src_reg = SRC,					\
193 194 195
		.off   = 0,					\
		.imm   = IMM })

196
/* Direct packet access, R0 = *(uint *) (skb->data + imm32) */
197

198
#define BPF_LD_ABS(SIZE, IMM)					\
199
	((struct bpf_insn) {					\
200
		.code  = BPF_LD | BPF_SIZE(SIZE) | BPF_ABS,	\
201 202
		.dst_reg = 0,					\
		.src_reg = 0,					\
203
		.off   = 0,					\
204
		.imm   = IMM })
205

206
/* Indirect packet access, R0 = *(uint *) (skb->data + src_reg + imm32) */
207

208
#define BPF_LD_IND(SIZE, SRC, IMM)				\
209
	((struct bpf_insn) {					\
210
		.code  = BPF_LD | BPF_SIZE(SIZE) | BPF_IND,	\
211 212
		.dst_reg = 0,					\
		.src_reg = SRC,					\
213
		.off   = 0,					\
214
		.imm   = IMM })
215

216
/* Memory load, dst_reg = *(uint *) (src_reg + off16) */
217

218
#define BPF_LDX_MEM(SIZE, DST, SRC, OFF)			\
219
	((struct bpf_insn) {					\
220
		.code  = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM,	\
221 222
		.dst_reg = DST,					\
		.src_reg = SRC,					\
223 224 225
		.off   = OFF,					\
		.imm   = 0 })

226 227 228
/* Memory store, *(uint *) (dst_reg + off16) = src_reg */

#define BPF_STX_MEM(SIZE, DST, SRC, OFF)			\
229
	((struct bpf_insn) {					\
230
		.code  = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM,	\
231 232
		.dst_reg = DST,					\
		.src_reg = SRC,					\
233 234 235
		.off   = OFF,					\
		.imm   = 0 })

236 237 238 239 240 241 242 243 244 245
/* Atomic memory add, *(uint *)(dst_reg + off16) += src_reg */

#define BPF_STX_XADD(SIZE, DST, SRC, OFF)			\
	((struct bpf_insn) {					\
		.code  = BPF_STX | BPF_SIZE(SIZE) | BPF_XADD,	\
		.dst_reg = DST,					\
		.src_reg = SRC,					\
		.off   = OFF,					\
		.imm   = 0 })

246 247 248
/* Memory store, *(uint *) (dst_reg + off16) = imm32 */

#define BPF_ST_MEM(SIZE, DST, OFF, IMM)				\
249
	((struct bpf_insn) {					\
250 251 252 253 254 255 256
		.code  = BPF_ST | BPF_SIZE(SIZE) | BPF_MEM,	\
		.dst_reg = DST,					\
		.src_reg = 0,					\
		.off   = OFF,					\
		.imm   = IMM })

/* Conditional jumps against registers, if (dst_reg 'op' src_reg) goto pc + off16 */
257

258
#define BPF_JMP_REG(OP, DST, SRC, OFF)				\
259
	((struct bpf_insn) {					\
260
		.code  = BPF_JMP | BPF_OP(OP) | BPF_X,		\
261 262
		.dst_reg = DST,					\
		.src_reg = SRC,					\
263 264 265
		.off   = OFF,					\
		.imm   = 0 })

266
/* Conditional jumps against immediates, if (dst_reg 'op' imm32) goto pc + off16 */
267

268
#define BPF_JMP_IMM(OP, DST, IMM, OFF)				\
269
	((struct bpf_insn) {					\
270
		.code  = BPF_JMP | BPF_OP(OP) | BPF_K,		\
271 272
		.dst_reg = DST,					\
		.src_reg = 0,					\
273 274 275
		.off   = OFF,					\
		.imm   = IMM })

276 277 278 279 280 281 282 283 284 285
/* Unconditional jumps, goto pc + off16 */

#define BPF_JMP_A(OFF)						\
	((struct bpf_insn) {					\
		.code  = BPF_JMP | BPF_JA,			\
		.dst_reg = 0,					\
		.src_reg = 0,					\
		.off   = OFF,					\
		.imm   = 0 })

286 287 288
/* Function call */

#define BPF_EMIT_CALL(FUNC)					\
289
	((struct bpf_insn) {					\
290
		.code  = BPF_JMP | BPF_CALL,			\
291 292
		.dst_reg = 0,					\
		.src_reg = 0,					\
293 294 295 296 297
		.off   = 0,					\
		.imm   = ((FUNC) - __bpf_call_base) })

/* Raw code statement block */

298
#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM)			\
299
	((struct bpf_insn) {					\
300
		.code  = CODE,					\
301 302
		.dst_reg = DST,					\
		.src_reg = SRC,					\
303 304 305 306 307 308
		.off   = OFF,					\
		.imm   = IMM })

/* Program exit */

#define BPF_EXIT_INSN()						\
309
	((struct bpf_insn) {					\
310
		.code  = BPF_JMP | BPF_EXIT,			\
311 312
		.dst_reg = 0,					\
		.src_reg = 0,					\
313 314 315
		.off   = 0,					\
		.imm   = 0 })

316 317 318 319 320 321 322 323
/* Internal classic blocks for direct assignment */

#define __BPF_STMT(CODE, K)					\
	((struct sock_filter) BPF_STMT(CODE, K))

#define __BPF_JUMP(CODE, K, JT, JF)				\
	((struct sock_filter) BPF_JUMP(CODE, K, JT, JF))

324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
#define bytes_to_bpf_size(bytes)				\
({								\
	int bpf_size = -EINVAL;					\
								\
	if (bytes == sizeof(u8))				\
		bpf_size = BPF_B;				\
	else if (bytes == sizeof(u16))				\
		bpf_size = BPF_H;				\
	else if (bytes == sizeof(u32))				\
		bpf_size = BPF_W;				\
	else if (bytes == sizeof(u64))				\
		bpf_size = BPF_DW;				\
								\
	bpf_size;						\
})
339

340 341 342 343 344 345 346 347 348 349 350 351 352 353
#define BPF_SIZEOF(type)					\
	({							\
		const int __size = bytes_to_bpf_size(sizeof(type)); \
		BUILD_BUG_ON(__size < 0);			\
		__size;						\
	})

#define BPF_FIELD_SIZEOF(type, field)				\
	({							\
		const int __size = bytes_to_bpf_size(FIELD_SIZEOF(type, field)); \
		BUILD_BUG_ON(__size < 0);			\
		__size;						\
	})

354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
#define __BPF_MAP_0(m, v, ...) v
#define __BPF_MAP_1(m, v, t, a, ...) m(t, a)
#define __BPF_MAP_2(m, v, t, a, ...) m(t, a), __BPF_MAP_1(m, v, __VA_ARGS__)
#define __BPF_MAP_3(m, v, t, a, ...) m(t, a), __BPF_MAP_2(m, v, __VA_ARGS__)
#define __BPF_MAP_4(m, v, t, a, ...) m(t, a), __BPF_MAP_3(m, v, __VA_ARGS__)
#define __BPF_MAP_5(m, v, t, a, ...) m(t, a), __BPF_MAP_4(m, v, __VA_ARGS__)

#define __BPF_REG_0(...) __BPF_PAD(5)
#define __BPF_REG_1(...) __BPF_MAP(1, __VA_ARGS__), __BPF_PAD(4)
#define __BPF_REG_2(...) __BPF_MAP(2, __VA_ARGS__), __BPF_PAD(3)
#define __BPF_REG_3(...) __BPF_MAP(3, __VA_ARGS__), __BPF_PAD(2)
#define __BPF_REG_4(...) __BPF_MAP(4, __VA_ARGS__), __BPF_PAD(1)
#define __BPF_REG_5(...) __BPF_MAP(5, __VA_ARGS__)

#define __BPF_MAP(n, ...) __BPF_MAP_##n(__VA_ARGS__)
#define __BPF_REG(n, ...) __BPF_REG_##n(__VA_ARGS__)

#define __BPF_CAST(t, a)						       \
	(__force t)							       \
	(__force							       \
	 typeof(__builtin_choose_expr(sizeof(t) == sizeof(unsigned long),      \
				      (unsigned long)0, (t)0))) a
#define __BPF_V void
#define __BPF_N

#define __BPF_DECL_ARGS(t, a) t   a
#define __BPF_DECL_REGS(t, a) u64 a

#define __BPF_PAD(n)							       \
	__BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2,       \
		  u64, __ur_3, u64, __ur_4, u64, __ur_5)

#define BPF_CALL_x(x, name, ...)					       \
	static __always_inline						       \
	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__));   \
	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__));	       \
	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__))	       \
	{								       \
		return ____##name(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
	}								       \
	static __always_inline						       \
	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))

#define BPF_CALL_0(name, ...)	BPF_CALL_x(0, name, __VA_ARGS__)
#define BPF_CALL_1(name, ...)	BPF_CALL_x(1, name, __VA_ARGS__)
#define BPF_CALL_2(name, ...)	BPF_CALL_x(2, name, __VA_ARGS__)
#define BPF_CALL_3(name, ...)	BPF_CALL_x(3, name, __VA_ARGS__)
#define BPF_CALL_4(name, ...)	BPF_CALL_x(4, name, __VA_ARGS__)
#define BPF_CALL_5(name, ...)	BPF_CALL_x(5, name, __VA_ARGS__)

404 405
#ifdef CONFIG_COMPAT
/* A struct sock_filter is architecture independent. */
406 407
struct compat_sock_fprog {
	u16		len;
408
	compat_uptr_t	filter;	/* struct sock_filter * */
409 410 411
};
#endif

412 413 414 415 416
struct sock_fprog_kern {
	u16			len;
	struct sock_filter	*filter;
};

417 418 419 420 421
struct bpf_binary_header {
	unsigned int pages;
	u8 image[];
};

422
struct bpf_prog {
423
	u16			pages;		/* Number of allocated pages */
424 425
	kmemcheck_bitfield_begin(meta);
	u16			jited:1,	/* Is our filter JIT'ed? */
426
				locked:1,	/* Program image locked? */
427
				gpl_compatible:1, /* Is filter GPL compatible? */
428
				cb_access:1,	/* Is control block accessed? */
429
				dst_needed:1;	/* Do we need dst entry? */
430
	kmemcheck_bitfield_end(meta);
431
	enum bpf_prog_type	type;		/* Type of BPF program */
432
	u32			len;		/* Number of filter blocks */
433
	u32			jited_len;	/* Size of jited insns in bytes */
434
	u8			tag[BPF_TAG_SIZE];
435
	struct bpf_prog_aux	*aux;		/* Auxiliary fields */
436
	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */
437 438
	unsigned int		(*bpf_func)(const void *ctx,
					    const struct bpf_insn *insn);
439
	/* Instructions for interpreter */
440
	union {
441
		struct sock_filter	insns[0];
442
		struct bpf_insn		insnsi[0];
443
	};
444 445
};

446
struct sk_filter {
447
	refcount_t	refcnt;
448 449 450 451 452 453
	struct rcu_head	rcu;
	struct bpf_prog	*prog;
};

#define BPF_PROG_RUN(filter, ctx)  (*filter->bpf_func)(ctx, filter->insnsi)

454 455
#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN

456 457 458 459 460
struct bpf_skb_data_end {
	struct qdisc_skb_cb qdisc_cb;
	void *data_end;
};

461 462 463
struct xdp_buff {
	void *data;
	void *data_end;
464
	void *data_hard_start;
465 466
};

467
/* compute the linear packet data range [data, data_end) which
468
 * will be accessed by cls_bpf, act_bpf and lwt programs
469 470 471 472 473 474 475 476 477
 */
static inline void bpf_compute_data_end(struct sk_buff *skb)
{
	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;

	BUILD_BUG_ON(sizeof(*cb) > FIELD_SIZEOF(struct sk_buff, cb));
	cb->data_end = skb->data + skb_headlen(skb);
}

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
static inline u8 *bpf_skb_cb(struct sk_buff *skb)
{
	/* eBPF programs may read/write skb->cb[] area to transfer meta
	 * data between tail calls. Since this also needs to work with
	 * tc, that scratch memory is mapped to qdisc_skb_cb's data area.
	 *
	 * In some socket filter cases, the cb unfortunately needs to be
	 * saved/restored so that protocol specific skb->cb[] data won't
	 * be lost. In any case, due to unpriviledged eBPF programs
	 * attached to sockets, we need to clear the bpf_skb_cb() area
	 * to not leak previous contents to user space.
	 */
	BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != BPF_SKB_CB_LEN);
	BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) !=
		     FIELD_SIZEOF(struct qdisc_skb_cb, data));

	return qdisc_skb_cb(skb)->data;
}

497 498 499
static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog,
				       struct sk_buff *skb)
{
500 501
	u8 *cb_data = bpf_skb_cb(skb);
	u8 cb_saved[BPF_SKB_CB_LEN];
502 503 504
	u32 res;

	if (unlikely(prog->cb_access)) {
505 506
		memcpy(cb_saved, cb_data, sizeof(cb_saved));
		memset(cb_data, 0, sizeof(cb_saved));
507 508 509 510 511
	}

	res = BPF_PROG_RUN(prog, skb);

	if (unlikely(prog->cb_access))
512
		memcpy(cb_data, cb_saved, sizeof(cb_saved));
513 514 515 516 517 518 519

	return res;
}

static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog,
					struct sk_buff *skb)
{
520
	u8 *cb_data = bpf_skb_cb(skb);
521 522

	if (unlikely(prog->cb_access))
523 524
		memset(cb_data, 0, BPF_SKB_CB_LEN);

525 526 527
	return BPF_PROG_RUN(prog, skb);
}

528 529
static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
					    struct xdp_buff *xdp)
530
{
531 532 533 534 535 536 537
	/* Caller needs to hold rcu_read_lock() (!), otherwise program
	 * can be released while still running, or map elements could be
	 * freed early while still having concurrent users. XDP fastpath
	 * already takes rcu_read_lock() when fetching the program, so
	 * it's not necessary here anymore.
	 */
	return BPF_PROG_RUN(prog, xdp);
538 539
}

540 541 542 543 544
static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
{
	return prog->len * sizeof(struct bpf_insn);
}

545
static inline u32 bpf_prog_tag_scratch_size(const struct bpf_prog *prog)
546 547 548 549 550
{
	return round_up(bpf_prog_insn_size(prog) +
			sizeof(__be64) + 1, SHA_MESSAGE_BYTES);
}

551
static inline unsigned int bpf_prog_size(unsigned int proglen)
552
{
553 554
	return max(sizeof(struct bpf_prog),
		   offsetof(struct bpf_prog, insns[proglen]));
555 556
}

557 558 559 560 561 562 563 564 565 566
static inline bool bpf_prog_was_classic(const struct bpf_prog *prog)
{
	/* When classic BPF programs have been loaded and the arch
	 * does not have a classic BPF JIT (anymore), they have been
	 * converted via bpf_migrate_filter() to eBPF and thus always
	 * have an unspec program type.
	 */
	return prog->type == BPF_PROG_TYPE_UNSPEC;
}

567
#define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0]))
568

569
#ifdef CONFIG_ARCH_HAS_SET_MEMORY
570 571
static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
{
572 573
	fp->locked = 1;
	WARN_ON_ONCE(set_memory_ro((unsigned long)fp, fp->pages));
574 575 576 577
}

static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
{
578 579 580 581 582 583 584
	if (fp->locked) {
		WARN_ON_ONCE(set_memory_rw((unsigned long)fp, fp->pages));
		/* In case set_memory_rw() fails, we want to be the first
		 * to crash here instead of some random place later on.
		 */
		fp->locked = 0;
	}
585
}
586

587 588
static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
{
589
	WARN_ON_ONCE(set_memory_ro((unsigned long)hdr, hdr->pages));
590 591
}

592 593
static inline void bpf_jit_binary_unlock_ro(struct bpf_binary_header *hdr)
{
594
	WARN_ON_ONCE(set_memory_rw((unsigned long)hdr, hdr->pages));
595
}
596 597 598 599 600 601 602 603
#else
static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
{
}

static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
{
}
604

605 606 607 608
static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
{
}

609 610 611
static inline void bpf_jit_binary_unlock_ro(struct bpf_binary_header *hdr)
{
}
612
#endif /* CONFIG_ARCH_HAS_SET_MEMORY */
613

614 615 616 617 618 619 620 621 622
static inline struct bpf_binary_header *
bpf_jit_binary_hdr(const struct bpf_prog *fp)
{
	unsigned long real_start = (unsigned long)fp->bpf_func;
	unsigned long addr = real_start & PAGE_MASK;

	return (void *)addr;
}

623 624 625 626 627
int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
{
	return sk_filter_trim_cap(sk, skb, 1);
}
628

629
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
630
void bpf_prog_free(struct bpf_prog *fp);
631

632 633 634 635 636 637 638 639 640 641 642
struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
				  gfp_t gfp_extra_flags);
void __bpf_prog_free(struct bpf_prog *fp);

static inline void bpf_prog_unlock_free(struct bpf_prog *fp)
{
	bpf_prog_unlock_ro(fp);
	__bpf_prog_free(fp);
}

643 644 645
typedef int (*bpf_aux_classic_check_t)(struct sock_filter *filter,
				       unsigned int flen);

646
int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog);
647
int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
648
			      bpf_aux_classic_check_t trans, bool save_orig);
649
void bpf_prog_destroy(struct bpf_prog *fp);
650

651
int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk);
652
int sk_attach_bpf(u32 ufd, struct sock *sk);
653 654
int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk);
int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk);
655 656 657 658
int sk_detach_filter(struct sock *sk);
int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,
		  unsigned int len);

659
bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
660
void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
661

662
u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
663 664

struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
665
void bpf_jit_compile(struct bpf_prog *prog);
666
bool bpf_helper_changes_pkt_data(void *func);
667

668 669
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
				       const struct bpf_insn *patch, u32 len);
670
void bpf_warn_invalid_xdp_action(u32 act);
671

672
#ifdef CONFIG_BPF_JIT
673
extern int bpf_jit_enable;
674
extern int bpf_jit_harden;
675
extern int bpf_jit_kallsyms;
676

677 678 679 680 681 682 683 684 685 686
typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);

struct bpf_binary_header *
bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
		     unsigned int alignment,
		     bpf_jit_fill_hole_t bpf_fill_ill_insns);
void bpf_jit_binary_free(struct bpf_binary_header *hdr);

void bpf_jit_free(struct bpf_prog *fp);

687 688 689
struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);

690 691 692
static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
				u32 pass, void *image)
{
693 694 695
	pr_err("flen=%u proglen=%u pass=%u image=%pK from=%s pid=%d\n", flen,
	       proglen, pass, image, current->comm, task_pid_nr(current));

696 697 698 699
	if (image)
		print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_OFFSET,
			       16, 1, image, proglen, false);
}
700 701 702 703 704 705 706 707 708 709

static inline bool bpf_jit_is_ebpf(void)
{
# ifdef CONFIG_HAVE_EBPF_JIT
	return true;
# else
	return false;
# endif
}

710 711 712 713 714
static inline bool ebpf_jit_enabled(void)
{
	return bpf_jit_enable && bpf_jit_is_ebpf();
}

715 716 717 718 719
static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp)
{
	return fp->jited && bpf_jit_is_ebpf();
}

720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736
static inline bool bpf_jit_blinding_enabled(void)
{
	/* These are the prerequisites, should someone ever have the
	 * idea to call blinding outside of them, we make sure to
	 * bail out.
	 */
	if (!bpf_jit_is_ebpf())
		return false;
	if (!bpf_jit_enable)
		return false;
	if (!bpf_jit_harden)
		return false;
	if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN))
		return false;

	return true;
}
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774

static inline bool bpf_jit_kallsyms_enabled(void)
{
	/* There are a couple of corner cases where kallsyms should
	 * not be enabled f.e. on hardening.
	 */
	if (bpf_jit_harden)
		return false;
	if (!bpf_jit_kallsyms)
		return false;
	if (bpf_jit_kallsyms == 1)
		return true;

	return false;
}

const char *__bpf_address_lookup(unsigned long addr, unsigned long *size,
				 unsigned long *off, char *sym);
bool is_bpf_text_address(unsigned long addr);
int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
		    char *sym);

static inline const char *
bpf_address_lookup(unsigned long addr, unsigned long *size,
		   unsigned long *off, char **modname, char *sym)
{
	const char *ret = __bpf_address_lookup(addr, size, off, sym);

	if (ret && modname)
		*modname = NULL;
	return ret;
}

void bpf_prog_kallsyms_add(struct bpf_prog *fp);
void bpf_prog_kallsyms_del(struct bpf_prog *fp);

#else /* CONFIG_BPF_JIT */

775 776 777 778 779
static inline bool ebpf_jit_enabled(void)
{
	return false;
}

780 781 782 783 784
static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp)
{
	return false;
}

785 786 787 788
static inline void bpf_jit_free(struct bpf_prog *fp)
{
	bpf_prog_unlock_free(fp);
}
789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826

static inline bool bpf_jit_kallsyms_enabled(void)
{
	return false;
}

static inline const char *
__bpf_address_lookup(unsigned long addr, unsigned long *size,
		     unsigned long *off, char *sym)
{
	return NULL;
}

static inline bool is_bpf_text_address(unsigned long addr)
{
	return false;
}

static inline int bpf_get_kallsym(unsigned int symnum, unsigned long *value,
				  char *type, char *sym)
{
	return -ERANGE;
}

static inline const char *
bpf_address_lookup(unsigned long addr, unsigned long *size,
		   unsigned long *off, char **modname, char *sym)
{
	return NULL;
}

static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp)
{
}

static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp)
{
}
827 828
#endif /* CONFIG_BPF_JIT */

829 830
#define BPF_ANC		BIT(15)

831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
static inline bool bpf_needs_clear_a(const struct sock_filter *first)
{
	switch (first->code) {
	case BPF_RET | BPF_K:
	case BPF_LD | BPF_W | BPF_LEN:
		return false;

	case BPF_LD | BPF_W | BPF_ABS:
	case BPF_LD | BPF_H | BPF_ABS:
	case BPF_LD | BPF_B | BPF_ABS:
		if (first->k == SKF_AD_OFF + SKF_AD_ALU_XOR_X)
			return true;
		return false;

	default:
		return true;
	}
}

850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
static inline u16 bpf_anc_helper(const struct sock_filter *ftest)
{
	BUG_ON(ftest->code & BPF_ANC);

	switch (ftest->code) {
	case BPF_LD | BPF_W | BPF_ABS:
	case BPF_LD | BPF_H | BPF_ABS:
	case BPF_LD | BPF_B | BPF_ABS:
#define BPF_ANCILLARY(CODE)	case SKF_AD_OFF + SKF_AD_##CODE:	\
				return BPF_ANC | SKF_AD_##CODE
		switch (ftest->k) {
		BPF_ANCILLARY(PROTOCOL);
		BPF_ANCILLARY(PKTTYPE);
		BPF_ANCILLARY(IFINDEX);
		BPF_ANCILLARY(NLATTR);
		BPF_ANCILLARY(NLATTR_NEST);
		BPF_ANCILLARY(MARK);
		BPF_ANCILLARY(QUEUE);
		BPF_ANCILLARY(HATYPE);
		BPF_ANCILLARY(RXHASH);
		BPF_ANCILLARY(CPU);
		BPF_ANCILLARY(ALU_XOR_X);
		BPF_ANCILLARY(VLAN_TAG);
		BPF_ANCILLARY(VLAN_TAG_PRESENT);
		BPF_ANCILLARY(PAY_OFFSET);
		BPF_ANCILLARY(RANDOM);
876
		BPF_ANCILLARY(VLAN_TPID);
877 878 879 880 881 882 883
		}
		/* Fallthrough. */
	default:
		return ftest->code;
	}
}

884 885 886 887 888 889 890 891 892 893 894 895
void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
					   int k, unsigned int size);

static inline void *bpf_load_pointer(const struct sk_buff *skb, int k,
				     unsigned int size, void *buffer)
{
	if (k >= 0)
		return skb_header_pointer(skb, k, size, buffer);

	return bpf_internal_load_pointer_neg_helper(skb, k, size);
}

896 897
static inline int bpf_tell_extensions(void)
{
898
	return SKF_AD_MAX;
899 900
}

L
Lawrence Brakmo 已提交
901 902 903 904 905 906 907 908 909
struct bpf_sock_ops_kern {
	struct	sock *sk;
	u32	op;
	union {
		u32 reply;
		u32 replylong[4];
	};
};

L
Linus Torvalds 已提交
910
#endif /* __LINUX_FILTER_H__ */