filter.h 11.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 * Linux Socket Filter Data Structures
 */
#ifndef __LINUX_FILTER_H__
#define __LINUX_FILTER_H__

A
Arun Sharma 已提交
7
#include <linux/atomic.h>
8
#include <linux/compat.h>
9
#include <linux/skbuff.h>
10
#include <linux/workqueue.h>
11
#include <uapi/linux/filter.h>
12

13 14
/* Internally used and optimized filter representation with extended
 * instruction set based on top of classic BPF.
15
 */
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40

/* instruction classes */
#define BPF_ALU64	0x07	/* alu mode in double word width */

/* ld/ldx fields */
#define BPF_DW		0x18	/* double word */
#define BPF_XADD	0xc0	/* exclusive add */

/* alu/jmp fields */
#define BPF_MOV		0xb0	/* mov reg to reg */
#define BPF_ARSH	0xc0	/* sign extending arithmetic shift right */

/* change endianness of a register */
#define BPF_END		0xd0	/* flags for endianness conversion: */
#define BPF_TO_LE	0x00	/* convert to little-endian */
#define BPF_TO_BE	0x08	/* convert to big-endian */
#define BPF_FROM_LE	BPF_TO_LE
#define BPF_FROM_BE	BPF_TO_BE

#define BPF_JNE		0x50	/* jump != */
#define BPF_JSGT	0x60	/* SGT is signed '>', GT in x86 */
#define BPF_JSGE	0x70	/* SGE is signed '>=', GE in x86 */
#define BPF_CALL	0x80	/* function call */
#define BPF_EXIT	0x90	/* function return */

41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
/* Register numbers */
enum {
	BPF_REG_0 = 0,
	BPF_REG_1,
	BPF_REG_2,
	BPF_REG_3,
	BPF_REG_4,
	BPF_REG_5,
	BPF_REG_6,
	BPF_REG_7,
	BPF_REG_8,
	BPF_REG_9,
	BPF_REG_10,
	__MAX_BPF_REG,
};

57
/* BPF has 10 general purpose 64-bit registers and stack frame. */
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
#define MAX_BPF_REG	__MAX_BPF_REG

/* ArgX, context and stack frame pointer register positions. Note,
 * Arg1, Arg2, Arg3, etc are used as argument mappings of function
 * calls in BPF_CALL instruction.
 */
#define BPF_REG_ARG1	BPF_REG_1
#define BPF_REG_ARG2	BPF_REG_2
#define BPF_REG_ARG3	BPF_REG_3
#define BPF_REG_ARG4	BPF_REG_4
#define BPF_REG_ARG5	BPF_REG_5
#define BPF_REG_CTX	BPF_REG_6
#define BPF_REG_FP	BPF_REG_10

/* Additional register mappings for converted user programs. */
#define BPF_REG_A	BPF_REG_0
#define BPF_REG_X	BPF_REG_7
#define BPF_REG_TMP	BPF_REG_8
76 77 78 79

/* BPF program can access up to 512 bytes of stack space. */
#define MAX_BPF_STACK	512

80 81
/* Helper macros for filter block array initializers. */

82
/* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
83

84
#define BPF_ALU64_REG(OP, DST, SRC)				\
85 86
	((struct sock_filter_int) {				\
		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_X,	\
87 88
		.dst_reg = DST,					\
		.src_reg = SRC,					\
89 90 91
		.off   = 0,					\
		.imm   = 0 })

92
#define BPF_ALU32_REG(OP, DST, SRC)				\
93 94
	((struct sock_filter_int) {				\
		.code  = BPF_ALU | BPF_OP(OP) | BPF_X,		\
95 96
		.dst_reg = DST,					\
		.src_reg = SRC,					\
97 98 99
		.off   = 0,					\
		.imm   = 0 })

100
/* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */
101

102
#define BPF_ALU64_IMM(OP, DST, IMM)				\
103 104
	((struct sock_filter_int) {				\
		.code  = BPF_ALU64 | BPF_OP(OP) | BPF_K,	\
105 106
		.dst_reg = DST,					\
		.src_reg = 0,					\
107 108 109
		.off   = 0,					\
		.imm   = IMM })

110
#define BPF_ALU32_IMM(OP, DST, IMM)				\
111 112
	((struct sock_filter_int) {				\
		.code  = BPF_ALU | BPF_OP(OP) | BPF_K,		\
113 114
		.dst_reg = DST,					\
		.src_reg = 0,					\
115 116 117 118 119
		.off   = 0,					\
		.imm   = IMM })

/* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */

120
#define BPF_ENDIAN(TYPE, DST, LEN)				\
121 122
	((struct sock_filter_int) {				\
		.code  = BPF_ALU | BPF_END | BPF_SRC(TYPE),	\
123 124
		.dst_reg = DST,					\
		.src_reg = 0,					\
125 126 127
		.off   = 0,					\
		.imm   = LEN })

128
/* Short form of mov, dst_reg = src_reg */
129

130
#define BPF_MOV64_REG(DST, SRC)					\
131 132
	((struct sock_filter_int) {				\
		.code  = BPF_ALU64 | BPF_MOV | BPF_X,		\
133 134
		.dst_reg = DST,					\
		.src_reg = SRC,					\
135 136 137
		.off   = 0,					\
		.imm   = 0 })

138
#define BPF_MOV32_REG(DST, SRC)					\
139 140
	((struct sock_filter_int) {				\
		.code  = BPF_ALU | BPF_MOV | BPF_X,		\
141 142
		.dst_reg = DST,					\
		.src_reg = SRC,					\
143 144 145
		.off   = 0,					\
		.imm   = 0 })

146
/* Short form of mov, dst_reg = imm32 */
147

148
#define BPF_MOV64_IMM(DST, IMM)					\
149 150
	((struct sock_filter_int) {				\
		.code  = BPF_ALU64 | BPF_MOV | BPF_K,		\
151 152
		.dst_reg = DST,					\
		.src_reg = 0,					\
153 154 155
		.off   = 0,					\
		.imm   = IMM })

156
#define BPF_MOV32_IMM(DST, IMM)					\
157 158
	((struct sock_filter_int) {				\
		.code  = BPF_ALU | BPF_MOV | BPF_K,		\
159 160
		.dst_reg = DST,					\
		.src_reg = 0,					\
161 162 163
		.off   = 0,					\
		.imm   = IMM })

164
/* Short form of mov based on type, BPF_X: dst_reg = src_reg, BPF_K: dst_reg = imm32 */
165

166
#define BPF_MOV64_RAW(TYPE, DST, SRC, IMM)			\
167 168
	((struct sock_filter_int) {				\
		.code  = BPF_ALU64 | BPF_MOV | BPF_SRC(TYPE),	\
169 170
		.dst_reg = DST,					\
		.src_reg = SRC,					\
171 172 173
		.off   = 0,					\
		.imm   = IMM })

174
#define BPF_MOV32_RAW(TYPE, DST, SRC, IMM)			\
175 176
	((struct sock_filter_int) {				\
		.code  = BPF_ALU | BPF_MOV | BPF_SRC(TYPE),	\
177 178
		.dst_reg = DST,					\
		.src_reg = SRC,					\
179 180 181
		.off   = 0,					\
		.imm   = IMM })

182
/* Direct packet access, R0 = *(uint *) (skb->data + imm32) */
183

184
#define BPF_LD_ABS(SIZE, IMM)					\
185 186
	((struct sock_filter_int) {				\
		.code  = BPF_LD | BPF_SIZE(SIZE) | BPF_ABS,	\
187 188
		.dst_reg = 0,					\
		.src_reg = 0,					\
189
		.off   = 0,					\
190
		.imm   = IMM })
191

192
/* Indirect packet access, R0 = *(uint *) (skb->data + src_reg + imm32) */
193

194
#define BPF_LD_IND(SIZE, SRC, IMM)				\
195 196
	((struct sock_filter_int) {				\
		.code  = BPF_LD | BPF_SIZE(SIZE) | BPF_IND,	\
197 198
		.dst_reg = 0,					\
		.src_reg = SRC,					\
199
		.off   = 0,					\
200
		.imm   = IMM })
201

202
/* Memory load, dst_reg = *(uint *) (src_reg + off16) */
203

204
#define BPF_LDX_MEM(SIZE, DST, SRC, OFF)			\
205 206
	((struct sock_filter_int) {				\
		.code  = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM,	\
207 208
		.dst_reg = DST,					\
		.src_reg = SRC,					\
209 210 211
		.off   = OFF,					\
		.imm   = 0 })

212 213 214
/* Memory store, *(uint *) (dst_reg + off16) = src_reg */

#define BPF_STX_MEM(SIZE, DST, SRC, OFF)			\
215 216
	((struct sock_filter_int) {				\
		.code  = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM,	\
217 218
		.dst_reg = DST,					\
		.src_reg = SRC,					\
219 220 221
		.off   = OFF,					\
		.imm   = 0 })

222 223 224 225 226 227 228 229 230 231 232
/* Memory store, *(uint *) (dst_reg + off16) = imm32 */

#define BPF_ST_MEM(SIZE, DST, OFF, IMM)				\
	((struct sock_filter_int) {				\
		.code  = BPF_ST | BPF_SIZE(SIZE) | BPF_MEM,	\
		.dst_reg = DST,					\
		.src_reg = 0,					\
		.off   = OFF,					\
		.imm   = IMM })

/* Conditional jumps against registers, if (dst_reg 'op' src_reg) goto pc + off16 */
233

234
#define BPF_JMP_REG(OP, DST, SRC, OFF)				\
235 236
	((struct sock_filter_int) {				\
		.code  = BPF_JMP | BPF_OP(OP) | BPF_X,		\
237 238
		.dst_reg = DST,					\
		.src_reg = SRC,					\
239 240 241
		.off   = OFF,					\
		.imm   = 0 })

242
/* Conditional jumps against immediates, if (dst_reg 'op' imm32) goto pc + off16 */
243

244
#define BPF_JMP_IMM(OP, DST, IMM, OFF)				\
245 246
	((struct sock_filter_int) {				\
		.code  = BPF_JMP | BPF_OP(OP) | BPF_K,		\
247 248
		.dst_reg = DST,					\
		.src_reg = 0,					\
249 250 251 252 253 254 255 256
		.off   = OFF,					\
		.imm   = IMM })

/* Function call */

#define BPF_EMIT_CALL(FUNC)					\
	((struct sock_filter_int) {				\
		.code  = BPF_JMP | BPF_CALL,			\
257 258
		.dst_reg = 0,					\
		.src_reg = 0,					\
259 260 261 262 263
		.off   = 0,					\
		.imm   = ((FUNC) - __bpf_call_base) })

/* Raw code statement block */

264
#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM)			\
265 266
	((struct sock_filter_int) {				\
		.code  = CODE,					\
267 268
		.dst_reg = DST,					\
		.src_reg = SRC,					\
269 270 271 272 273 274 275 276
		.off   = OFF,					\
		.imm   = IMM })

/* Program exit */

#define BPF_EXIT_INSN()						\
	((struct sock_filter_int) {				\
		.code  = BPF_JMP | BPF_EXIT,			\
277 278
		.dst_reg = 0,					\
		.src_reg = 0,					\
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
		.off   = 0,					\
		.imm   = 0 })

#define bytes_to_bpf_size(bytes)				\
({								\
	int bpf_size = -EINVAL;					\
								\
	if (bytes == sizeof(u8))				\
		bpf_size = BPF_B;				\
	else if (bytes == sizeof(u16))				\
		bpf_size = BPF_H;				\
	else if (bytes == sizeof(u32))				\
		bpf_size = BPF_W;				\
	else if (bytes == sizeof(u64))				\
		bpf_size = BPF_DW;				\
								\
	bpf_size;						\
})
297

298 299
/* Macro to invoke filter function. */
#define SK_RUN_FILTER(filter, ctx)  (*filter->bpf_func)(ctx, filter->insnsi)
300 301 302

struct sock_filter_int {
	__u8	code;		/* opcode */
303 304
	__u8	dst_reg:4;	/* dest register */
	__u8	src_reg:4;	/* source register */
305 306 307 308 309 310
	__s16	off;		/* signed offset */
	__s32	imm;		/* signed immediate constant */
};

#ifdef CONFIG_COMPAT
/* A struct sock_filter is architecture independent. */
311 312
struct compat_sock_fprog {
	u16		len;
313
	compat_uptr_t	filter;	/* struct sock_filter * */
314 315 316
};
#endif

317 318 319 320 321
struct sock_fprog_kern {
	u16			len;
	struct sock_filter	*filter;
};

322 323
struct sk_buff;
struct sock;
324
struct seccomp_data;
325

326
struct sk_filter {
327
	atomic_t		refcnt;
328 329
	u32			jited:1,	/* Is our filter JIT'ed? */
				len:31;		/* Number of filter blocks */
330
	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */
331
	struct rcu_head		rcu;
332
	unsigned int		(*bpf_func)(const struct sk_buff *skb,
333
					    const struct sock_filter_int *filter);
334
	union {
335 336
		struct sock_filter	insns[0];
		struct sock_filter_int	insnsi[0];
337 338
		struct work_struct	work;
	};
339 340
};

341
static inline unsigned int sk_filter_size(unsigned int proglen)
342
{
343 344
	return max(sizeof(struct sk_filter),
		   offsetof(struct sk_filter, insns[proglen]));
345 346
}

347 348 349
#define sk_filter_proglen(fprog)			\
		(fprog->len * sizeof(fprog->filter[0]))

350
int sk_filter(struct sock *sk, struct sk_buff *skb);
351

352 353
void sk_filter_select_runtime(struct sk_filter *fp);
void sk_filter_free(struct sk_filter *fp);
354 355 356

int sk_convert_filter(struct sock_filter *prog, int len,
		      struct sock_filter_int *new_prog, int *new_len);
357

358
int sk_unattached_filter_create(struct sk_filter **pfp,
359
				struct sock_fprog_kern *fprog);
360
void sk_unattached_filter_destroy(struct sk_filter *fp);
361

362 363
int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk);
int sk_detach_filter(struct sock *sk);
364

365 366 367 368 369 370
int sk_chk_filter(struct sock_filter *filter, unsigned int flen);
int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,
		  unsigned int len);

void sk_filter_charge(struct sock *sk, struct sk_filter *fp);
void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
371

372 373 374
u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
void bpf_int_jit_compile(struct sk_filter *fp);

375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
#define BPF_ANC		BIT(15)

static inline u16 bpf_anc_helper(const struct sock_filter *ftest)
{
	BUG_ON(ftest->code & BPF_ANC);

	switch (ftest->code) {
	case BPF_LD | BPF_W | BPF_ABS:
	case BPF_LD | BPF_H | BPF_ABS:
	case BPF_LD | BPF_B | BPF_ABS:
#define BPF_ANCILLARY(CODE)	case SKF_AD_OFF + SKF_AD_##CODE:	\
				return BPF_ANC | SKF_AD_##CODE
		switch (ftest->k) {
		BPF_ANCILLARY(PROTOCOL);
		BPF_ANCILLARY(PKTTYPE);
		BPF_ANCILLARY(IFINDEX);
		BPF_ANCILLARY(NLATTR);
		BPF_ANCILLARY(NLATTR_NEST);
		BPF_ANCILLARY(MARK);
		BPF_ANCILLARY(QUEUE);
		BPF_ANCILLARY(HATYPE);
		BPF_ANCILLARY(RXHASH);
		BPF_ANCILLARY(CPU);
		BPF_ANCILLARY(ALU_XOR_X);
		BPF_ANCILLARY(VLAN_TAG);
		BPF_ANCILLARY(VLAN_TAG_PRESENT);
		BPF_ANCILLARY(PAY_OFFSET);
		BPF_ANCILLARY(RANDOM);
		}
		/* Fallthrough. */
	default:
		return ftest->code;
	}
}

410 411 412 413 414 415 416 417 418 419 420 421
void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
					   int k, unsigned int size);

static inline void *bpf_load_pointer(const struct sk_buff *skb, int k,
				     unsigned int size, void *buffer)
{
	if (k >= 0)
		return skb_header_pointer(skb, k, size, buffer);

	return bpf_internal_load_pointer_neg_helper(skb, k, size);
}

422
#ifdef CONFIG_BPF_JIT
X
Xi Wang 已提交
423
#include <stdarg.h>
424 425 426
#include <linux/linkage.h>
#include <linux/printk.h>

427 428
void bpf_jit_compile(struct sk_filter *fp);
void bpf_jit_free(struct sk_filter *fp);
429 430 431 432

static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
				u32 pass, void *image)
{
433
	pr_err("flen=%u proglen=%u pass=%u image=%pK\n",
434 435
	       flen, proglen, pass, image);
	if (image)
436
		print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_OFFSET,
437 438
			       16, 1, image, proglen, false);
}
439
#else
440
#include <linux/slab.h>
441

442 443 444
static inline void bpf_jit_compile(struct sk_filter *fp)
{
}
445

446 447
static inline void bpf_jit_free(struct sk_filter *fp)
{
448
	kfree(fp);
449
}
450
#endif /* CONFIG_BPF_JIT */
451

452 453
static inline int bpf_tell_extensions(void)
{
454
	return SKF_AD_MAX;
455 456
}

L
Linus Torvalds 已提交
457
#endif /* __LINUX_FILTER_H__ */