core.c 45.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/*
 * Linux Socket Filter - Kernel level socket filtering
 *
 * Based on the design of the Berkeley Packet Filter. The new
 * internal format has been designed by PLUMgrid:
 *
 *	Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
 *
 * Authors:
 *
 *	Jay Schulist <jschlst@samba.org>
 *	Alexei Starovoitov <ast@plumgrid.com>
 *	Daniel Borkmann <dborkman@redhat.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 *
 * Andi Kleen - Fix a few bad bugs and races.
21
 * Kris Katterjohn - Added many additional checks in bpf_check_classic()
22
 */
23

24 25
#include <linux/filter.h>
#include <linux/skbuff.h>
26
#include <linux/vmalloc.h>
27 28
#include <linux/random.h>
#include <linux/moduleloader.h>
29
#include <linux/bpf.h>
30
#include <linux/frame.h>
31 32 33
#include <linux/rbtree_latch.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
Y
Yonghong Song 已提交
34
#include <linux/perf_event.h>
35

D
Daniel Borkmann 已提交
36 37
#include <asm/unaligned.h>

38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
/* Registers */
#define BPF_R0	regs[BPF_REG_0]
#define BPF_R1	regs[BPF_REG_1]
#define BPF_R2	regs[BPF_REG_2]
#define BPF_R3	regs[BPF_REG_3]
#define BPF_R4	regs[BPF_REG_4]
#define BPF_R5	regs[BPF_REG_5]
#define BPF_R6	regs[BPF_REG_6]
#define BPF_R7	regs[BPF_REG_7]
#define BPF_R8	regs[BPF_REG_8]
#define BPF_R9	regs[BPF_REG_9]
#define BPF_R10	regs[BPF_REG_10]

/* Named registers */
#define DST	regs[insn->dst_reg]
#define SRC	regs[insn->src_reg]
#define FP	regs[BPF_REG_FP]
#define ARG1	regs[BPF_REG_ARG1]
#define CTX	regs[BPF_REG_CTX]
#define IMM	insn->imm

/* No hurry in this branch
 *
 * Exported for the bpf jit load helper.
 */
void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size)
{
	u8 *ptr = NULL;

	if (k >= SKF_NET_OFF)
		ptr = skb_network_header(skb) + k - SKF_NET_OFF;
	else if (k >= SKF_LL_OFF)
		ptr = skb_mac_header(skb) + k - SKF_LL_OFF;
D
Daniel Borkmann 已提交
71

72 73 74 75 76 77
	if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))
		return ptr;

	return NULL;
}

78 79
struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
{
80
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
81
	struct bpf_prog_aux *aux;
82 83 84 85 86 87 88
	struct bpf_prog *fp;

	size = round_up(size, PAGE_SIZE);
	fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
	if (fp == NULL)
		return NULL;

89 90
	aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags);
	if (aux == NULL) {
91 92 93 94 95
		vfree(fp);
		return NULL;
	}

	fp->pages = size / PAGE_SIZE;
96
	fp->aux = aux;
97
	fp->aux->prog = fp;
98
	fp->jit_requested = ebpf_jit_enabled();
99

100 101
	INIT_LIST_HEAD_RCU(&fp->aux->ksym_lnode);

102 103 104 105 106 107 108
	return fp;
}
EXPORT_SYMBOL_GPL(bpf_prog_alloc);

struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
				  gfp_t gfp_extra_flags)
{
109
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
110
	struct bpf_prog *fp;
111 112
	u32 pages, delta;
	int ret;
113 114 115 116

	BUG_ON(fp_old == NULL);

	size = round_up(size, PAGE_SIZE);
117 118
	pages = size / PAGE_SIZE;
	if (pages <= fp_old->pages)
119 120
		return fp_old;

121 122 123 124 125
	delta = pages - fp_old->pages;
	ret = __bpf_prog_charge(fp_old->aux->user, delta);
	if (ret)
		return NULL;

126
	fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
127 128 129
	if (fp == NULL) {
		__bpf_prog_uncharge(fp_old->aux->user, delta);
	} else {
130
		memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);
131
		fp->pages = pages;
132
		fp->aux->prog = fp;
133

134
		/* We keep fp->aux from fp_old around in the new
135 136
		 * reallocated structure.
		 */
137
		fp_old->aux = NULL;
138 139 140 141 142 143 144 145
		__bpf_prog_free(fp_old);
	}

	return fp;
}

void __bpf_prog_free(struct bpf_prog *fp)
{
146
	kfree(fp->aux);
147 148 149
	vfree(fp);
}

150
int bpf_prog_calc_tag(struct bpf_prog *fp)
151 152
{
	const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64);
153 154
	u32 raw_size = bpf_prog_tag_scratch_size(fp);
	u32 digest[SHA_DIGEST_WORDS];
155
	u32 ws[SHA_WORKSPACE_WORDS];
156
	u32 i, bsize, psize, blocks;
157
	struct bpf_insn *dst;
158
	bool was_ld_map;
159
	u8 *raw, *todo;
160 161 162
	__be32 *result;
	__be64 *bits;

163 164 165 166
	raw = vmalloc(raw_size);
	if (!raw)
		return -ENOMEM;

167
	sha_init(digest);
168 169 170 171 172
	memset(ws, 0, sizeof(ws));

	/* We need to take out the map fd for the digest calculation
	 * since they are unstable from user space side.
	 */
173
	dst = (void *)raw;
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
	for (i = 0, was_ld_map = false; i < fp->len; i++) {
		dst[i] = fp->insnsi[i];
		if (!was_ld_map &&
		    dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) &&
		    dst[i].src_reg == BPF_PSEUDO_MAP_FD) {
			was_ld_map = true;
			dst[i].imm = 0;
		} else if (was_ld_map &&
			   dst[i].code == 0 &&
			   dst[i].dst_reg == 0 &&
			   dst[i].src_reg == 0 &&
			   dst[i].off == 0) {
			was_ld_map = false;
			dst[i].imm = 0;
		} else {
			was_ld_map = false;
		}
	}

193 194
	psize = bpf_prog_insn_size(fp);
	memset(&raw[psize], 0, raw_size - psize);
195 196 197 198
	raw[psize++] = 0x80;

	bsize  = round_up(psize, SHA_MESSAGE_BYTES);
	blocks = bsize / SHA_MESSAGE_BYTES;
199
	todo   = raw;
200 201 202 203 204 205 206 207 208
	if (bsize - psize >= sizeof(__be64)) {
		bits = (__be64 *)(todo + bsize - sizeof(__be64));
	} else {
		bits = (__be64 *)(todo + bsize + bits_offset);
		blocks++;
	}
	*bits = cpu_to_be64((psize - 1) << 3);

	while (blocks--) {
209
		sha_transform(digest, todo, ws);
210 211 212
		todo += SHA_MESSAGE_BYTES;
	}

213
	result = (__force __be32 *)digest;
214
	for (i = 0; i < SHA_DIGEST_WORDS; i++)
215 216
		result[i] = cpu_to_be32(digest[i]);
	memcpy(fp->tag, result, sizeof(fp->tag));
217 218 219

	vfree(raw);
	return 0;
220 221
}

222 223
static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, u32 delta,
				u32 curr, const bool probe_pass)
224
{
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
	const s64 imm_min = S32_MIN, imm_max = S32_MAX;
	s64 imm = insn->imm;

	if (curr < pos && curr + imm + 1 > pos)
		imm += delta;
	else if (curr > pos + delta && curr + imm + 1 <= pos + delta)
		imm -= delta;
	if (imm < imm_min || imm > imm_max)
		return -ERANGE;
	if (!probe_pass)
		insn->imm = imm;
	return 0;
}

static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, u32 delta,
				u32 curr, const bool probe_pass)
{
	const s32 off_min = S16_MIN, off_max = S16_MAX;
	s32 off = insn->off;

	if (curr < pos && curr + off + 1 > pos)
		off += delta;
	else if (curr > pos + delta && curr + off + 1 <= pos + delta)
		off -= delta;
	if (off < off_min || off > off_max)
		return -ERANGE;
	if (!probe_pass)
		insn->off = off;
	return 0;
}

static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta,
			    const bool probe_pass)
{
	u32 i, insn_cnt = prog->len + (probe_pass ? delta : 0);
260
	struct bpf_insn *insn = prog->insnsi;
261
	int ret = 0;
262 263

	for (i = 0; i < insn_cnt; i++, insn++) {
264 265 266 267 268 269 270 271 272 273
		u8 code;

		/* In the probing pass we still operate on the original,
		 * unpatched image in order to check overflows before we
		 * do any other adjustments. Therefore skip the patchlet.
		 */
		if (probe_pass && i == pos) {
			i += delta + 1;
			insn++;
		}
274
		code = insn->code;
275 276
		if (BPF_CLASS(code) != BPF_JMP ||
		    BPF_OP(code) == BPF_EXIT)
277
			continue;
278
		/* Adjust offset of jmps if we cross patch boundaries. */
279
		if (BPF_OP(code) == BPF_CALL) {
280
			if (insn->src_reg != BPF_PSEUDO_CALL)
281
				continue;
282 283
			ret = bpf_adj_delta_to_imm(insn, pos, delta, i,
						   probe_pass);
284
		} else {
285 286
			ret = bpf_adj_delta_to_off(insn, pos, delta, i,
						   probe_pass);
287
		}
288 289
		if (ret)
			break;
290
	}
291 292

	return ret;
293 294 295 296 297 298
}

struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
				       const struct bpf_insn *patch, u32 len)
{
	u32 insn_adj_cnt, insn_rest, insn_delta = len - 1;
299
	const u32 cnt_max = S16_MAX;
300 301 302 303 304 305 306 307 308 309
	struct bpf_prog *prog_adj;

	/* Since our patchlet doesn't expand the image, we're done. */
	if (insn_delta == 0) {
		memcpy(prog->insnsi + off, patch, sizeof(*patch));
		return prog;
	}

	insn_adj_cnt = prog->len + insn_delta;

310 311 312 313 314 315 316 317 318
	/* Reject anything that would potentially let the insn->off
	 * target overflow when we have excessive program expansions.
	 * We need to probe here before we do any reallocation where
	 * we afterwards may not fail anymore.
	 */
	if (insn_adj_cnt > cnt_max &&
	    bpf_adj_branches(prog, off, insn_delta, true))
		return NULL;

319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
	/* Several new instructions need to be inserted. Make room
	 * for them. Likely, there's no need for a new allocation as
	 * last page could have large enough tailroom.
	 */
	prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),
				    GFP_USER);
	if (!prog_adj)
		return NULL;

	prog_adj->len = insn_adj_cnt;

	/* Patching happens in 3 steps:
	 *
	 * 1) Move over tail of insnsi from next instruction onwards,
	 *    so we can patch the single target insn with one or more
	 *    new ones (patching is always from 1 to n insns, n > 0).
	 * 2) Inject new instructions at the target location.
	 * 3) Adjust branch offsets if necessary.
	 */
	insn_rest = insn_adj_cnt - off - len;

	memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1,
		sizeof(*patch) * insn_rest);
	memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len);

344 345 346 347 348
	/* We are guaranteed to not fail at this point, otherwise
	 * the ship has sailed to reverse to the original state. An
	 * overflow cannot happen at this point.
	 */
	BUG_ON(bpf_adj_branches(prog_adj, off, insn_delta, false));
349 350 351 352

	return prog_adj;
}

353 354 355 356 357 358 359 360 361 362 363 364 365 366
void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp)
{
	int i;

	for (i = 0; i < fp->aux->func_cnt; i++)
		bpf_prog_kallsyms_del(fp->aux->func[i]);
}

void bpf_prog_kallsyms_del_all(struct bpf_prog *fp)
{
	bpf_prog_kallsyms_del_subprogs(fp);
	bpf_prog_kallsyms_del(fp);
}

367
#ifdef CONFIG_BPF_JIT
368 369 370 371 372
/* All BPF JIT sysctl knobs here. */
int bpf_jit_enable   __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON);
int bpf_jit_harden   __read_mostly;
int bpf_jit_kallsyms __read_mostly;

373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
static __always_inline void
bpf_get_prog_addr_region(const struct bpf_prog *prog,
			 unsigned long *symbol_start,
			 unsigned long *symbol_end)
{
	const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog);
	unsigned long addr = (unsigned long)hdr;

	WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog));

	*symbol_start = addr;
	*symbol_end   = addr + hdr->pages * PAGE_SIZE;
}

static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)
{
389 390
	const char *end = sym + KSYM_NAME_LEN;

391
	BUILD_BUG_ON(sizeof("bpf_prog_") +
392 393 394 395 396 397 398 399 400
		     sizeof(prog->tag) * 2 +
		     /* name has been null terminated.
		      * We should need +1 for the '_' preceding
		      * the name.  However, the null character
		      * is double counted between the name and the
		      * sizeof("bpf_prog_") above, so we omit
		      * the +1 here.
		      */
		     sizeof(prog->aux->name) > KSYM_NAME_LEN);
401 402 403

	sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_");
	sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));
404 405 406 407
	if (prog->aux->name[0])
		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);
	else
		*sym = 0;
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
}

static __always_inline unsigned long
bpf_get_prog_addr_start(struct latch_tree_node *n)
{
	unsigned long symbol_start, symbol_end;
	const struct bpf_prog_aux *aux;

	aux = container_of(n, struct bpf_prog_aux, ksym_tnode);
	bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);

	return symbol_start;
}

static __always_inline bool bpf_tree_less(struct latch_tree_node *a,
					  struct latch_tree_node *b)
{
	return bpf_get_prog_addr_start(a) < bpf_get_prog_addr_start(b);
}

static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n)
{
	unsigned long val = (unsigned long)key;
	unsigned long symbol_start, symbol_end;
	const struct bpf_prog_aux *aux;

	aux = container_of(n, struct bpf_prog_aux, ksym_tnode);
	bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);

	if (val < symbol_start)
		return -1;
	if (val >= symbol_end)
		return  1;

	return 0;
}

static const struct latch_tree_ops bpf_tree_ops = {
	.less	= bpf_tree_less,
	.comp	= bpf_tree_comp,
};

static DEFINE_SPINLOCK(bpf_lock);
static LIST_HEAD(bpf_kallsyms);
static struct latch_tree_root bpf_tree __cacheline_aligned;

static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux)
{
	WARN_ON_ONCE(!list_empty(&aux->ksym_lnode));
	list_add_tail_rcu(&aux->ksym_lnode, &bpf_kallsyms);
	latch_tree_insert(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops);
}

static void bpf_prog_ksym_node_del(struct bpf_prog_aux *aux)
{
	if (list_empty(&aux->ksym_lnode))
		return;

	latch_tree_erase(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops);
	list_del_rcu(&aux->ksym_lnode);
}

static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp)
{
	return fp->jited && !bpf_prog_was_classic(fp);
}

static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
{
	return list_empty(&fp->aux->ksym_lnode) ||
	       fp->aux->ksym_lnode.prev == LIST_POISON2;
}

void bpf_prog_kallsyms_add(struct bpf_prog *fp)
{
	if (!bpf_prog_kallsyms_candidate(fp) ||
	    !capable(CAP_SYS_ADMIN))
		return;

487
	spin_lock_bh(&bpf_lock);
488
	bpf_prog_ksym_node_add(fp->aux);
489
	spin_unlock_bh(&bpf_lock);
490 491 492 493 494 495 496
}

void bpf_prog_kallsyms_del(struct bpf_prog *fp)
{
	if (!bpf_prog_kallsyms_candidate(fp))
		return;

497
	spin_lock_bh(&bpf_lock);
498
	bpf_prog_ksym_node_del(fp->aux);
499
	spin_unlock_bh(&bpf_lock);
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
}

static struct bpf_prog *bpf_prog_kallsyms_find(unsigned long addr)
{
	struct latch_tree_node *n;

	if (!bpf_jit_kallsyms_enabled())
		return NULL;

	n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops);
	return n ?
	       container_of(n, struct bpf_prog_aux, ksym_tnode)->prog :
	       NULL;
}

const char *__bpf_address_lookup(unsigned long addr, unsigned long *size,
				 unsigned long *off, char *sym)
{
	unsigned long symbol_start, symbol_end;
	struct bpf_prog *prog;
	char *ret = NULL;

	rcu_read_lock();
	prog = bpf_prog_kallsyms_find(addr);
	if (prog) {
		bpf_get_prog_addr_region(prog, &symbol_start, &symbol_end);
		bpf_get_prog_name(prog, sym);

		ret = sym;
		if (size)
			*size = symbol_end - symbol_start;
		if (off)
			*off  = addr - symbol_start;
	}
	rcu_read_unlock();

	return ret;
}

bool is_bpf_text_address(unsigned long addr)
{
	bool ret;

	rcu_read_lock();
	ret = bpf_prog_kallsyms_find(addr) != NULL;
	rcu_read_unlock();

	return ret;
}

int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
		    char *sym)
{
	unsigned long symbol_start, symbol_end;
	struct bpf_prog_aux *aux;
	unsigned int it = 0;
	int ret = -ERANGE;

	if (!bpf_jit_kallsyms_enabled())
		return ret;

	rcu_read_lock();
	list_for_each_entry_rcu(aux, &bpf_kallsyms, ksym_lnode) {
		if (it++ != symnum)
			continue;

		bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);
		bpf_get_prog_name(aux->prog, sym);

		*value = symbol_start;
		*type  = BPF_SYM_ELF_TYPE;

		ret = 0;
		break;
	}
	rcu_read_unlock();

	return ret;
}

580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
struct bpf_binary_header *
bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
		     unsigned int alignment,
		     bpf_jit_fill_hole_t bpf_fill_ill_insns)
{
	struct bpf_binary_header *hdr;
	unsigned int size, hole, start;

	/* Most of BPF filters are really small, but if some of them
	 * fill a page, allow at least 128 extra bytes to insert a
	 * random section of illegal instructions.
	 */
	size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE);
	hdr = module_alloc(size);
	if (hdr == NULL)
		return NULL;

	/* Fill space with illegal/arch-dep instructions. */
	bpf_fill_ill_insns(hdr, size);

	hdr->pages = size / PAGE_SIZE;
	hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)),
		     PAGE_SIZE - sizeof(*hdr));
603
	start = (get_random_int() % hole) & ~(alignment - 1);
604 605 606 607 608 609 610 611 612

	/* Leave a random number of instructions before BPF code. */
	*image_ptr = &hdr->image[start];

	return hdr;
}

void bpf_jit_binary_free(struct bpf_binary_header *hdr)
{
613
	module_memfree(hdr);
614
}
615

616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
/* This symbol is only overridden by archs that have different
 * requirements than the usual eBPF JITs, f.e. when they only
 * implement cBPF JIT, do not set images read-only, etc.
 */
void __weak bpf_jit_free(struct bpf_prog *fp)
{
	if (fp->jited) {
		struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp);

		bpf_jit_binary_unlock_ro(hdr);
		bpf_jit_binary_free(hdr);

		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
	}

	bpf_prog_unlock_free(fp);
}

634 635 636 637 638
static int bpf_jit_blind_insn(const struct bpf_insn *from,
			      const struct bpf_insn *aux,
			      struct bpf_insn *to_buff)
{
	struct bpf_insn *to = to_buff;
639
	u32 imm_rnd = get_random_int();
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
	s16 off;

	BUILD_BUG_ON(BPF_REG_AX  + 1 != MAX_BPF_JIT_REG);
	BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG);

	if (from->imm == 0 &&
	    (from->code == (BPF_ALU   | BPF_MOV | BPF_K) ||
	     from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) {
		*to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg);
		goto out;
	}

	switch (from->code) {
	case BPF_ALU | BPF_ADD | BPF_K:
	case BPF_ALU | BPF_SUB | BPF_K:
	case BPF_ALU | BPF_AND | BPF_K:
	case BPF_ALU | BPF_OR  | BPF_K:
	case BPF_ALU | BPF_XOR | BPF_K:
	case BPF_ALU | BPF_MUL | BPF_K:
	case BPF_ALU | BPF_MOV | BPF_K:
	case BPF_ALU | BPF_DIV | BPF_K:
	case BPF_ALU | BPF_MOD | BPF_K:
		*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
		*to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
		*to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX);
		break;

	case BPF_ALU64 | BPF_ADD | BPF_K:
	case BPF_ALU64 | BPF_SUB | BPF_K:
	case BPF_ALU64 | BPF_AND | BPF_K:
	case BPF_ALU64 | BPF_OR  | BPF_K:
	case BPF_ALU64 | BPF_XOR | BPF_K:
	case BPF_ALU64 | BPF_MUL | BPF_K:
	case BPF_ALU64 | BPF_MOV | BPF_K:
	case BPF_ALU64 | BPF_DIV | BPF_K:
	case BPF_ALU64 | BPF_MOD | BPF_K:
		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
		*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
		*to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX);
		break;

	case BPF_JMP | BPF_JEQ  | BPF_K:
	case BPF_JMP | BPF_JNE  | BPF_K:
	case BPF_JMP | BPF_JGT  | BPF_K:
684
	case BPF_JMP | BPF_JLT  | BPF_K:
685
	case BPF_JMP | BPF_JGE  | BPF_K:
686
	case BPF_JMP | BPF_JLE  | BPF_K:
687
	case BPF_JMP | BPF_JSGT | BPF_K:
688
	case BPF_JMP | BPF_JSLT | BPF_K:
689
	case BPF_JMP | BPF_JSGE | BPF_K:
690
	case BPF_JMP | BPF_JSLE | BPF_K:
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
	case BPF_JMP | BPF_JSET | BPF_K:
		/* Accommodate for extra offset in case of a backjump. */
		off = from->off;
		if (off < 0)
			off -= 2;
		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
		*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
		*to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off);
		break;

	case BPF_LD | BPF_IMM | BPF_DW:
		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm);
		*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
		*to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
		*to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX);
		break;
	case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */
		*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm);
		*to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
		*to++ = BPF_ALU64_REG(BPF_OR,  aux[0].dst_reg, BPF_REG_AX);
		break;

	case BPF_ST | BPF_MEM | BPF_DW:
	case BPF_ST | BPF_MEM | BPF_W:
	case BPF_ST | BPF_MEM | BPF_H:
	case BPF_ST | BPF_MEM | BPF_B:
		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
		*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
		*to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off);
		break;
	}
out:
	return to - to_buff;
}

static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other,
					      gfp_t gfp_extra_flags)
{
729
	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
	struct bpf_prog *fp;

	fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags, PAGE_KERNEL);
	if (fp != NULL) {
		/* aux->prog still points to the fp_other one, so
		 * when promoting the clone to the real program,
		 * this still needs to be adapted.
		 */
		memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE);
	}

	return fp;
}

static void bpf_prog_clone_free(struct bpf_prog *fp)
{
	/* aux was stolen by the other clone, so we cannot free
	 * it from this path! It will be freed eventually by the
	 * other program on release.
	 *
	 * At this point, we don't need a deferred release since
	 * clone is guaranteed to not be locked.
	 */
	fp->aux = NULL;
	__bpf_prog_free(fp);
}

void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
{
	/* We have to repoint aux->prog to self, as we don't
	 * know whether fp here is the clone or the original.
	 */
	fp->aux->prog = fp;
	bpf_prog_clone_free(fp_other);
}

struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
{
	struct bpf_insn insn_buff[16], aux[2];
	struct bpf_prog *clone, *tmp;
	int insn_delta, insn_cnt;
	struct bpf_insn *insn;
	int i, rewritten;

774
	if (!bpf_jit_blinding_enabled(prog) || prog->blinded)
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
		return prog;

	clone = bpf_prog_clone_create(prog, GFP_USER);
	if (!clone)
		return ERR_PTR(-ENOMEM);

	insn_cnt = clone->len;
	insn = clone->insnsi;

	for (i = 0; i < insn_cnt; i++, insn++) {
		/* We temporarily need to hold the original ld64 insn
		 * so that we can still access the first part in the
		 * second blinding run.
		 */
		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) &&
		    insn[1].code == 0)
			memcpy(aux, insn, sizeof(aux));

		rewritten = bpf_jit_blind_insn(insn, aux, insn_buff);
		if (!rewritten)
			continue;

		tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
		if (!tmp) {
			/* Patching may have repointed aux->prog during
			 * realloc from the original one, so we need to
			 * fix it up here on error.
			 */
			bpf_jit_prog_release_other(prog, clone);
			return ERR_PTR(-ENOMEM);
		}

		clone = tmp;
		insn_delta = rewritten - 1;

		/* Walk new program and skip insns we just inserted. */
		insn = clone->insnsi + i + insn_delta;
		insn_cnt += insn_delta;
		i        += insn_delta;
	}

816
	clone->blinded = 1;
817 818
	return clone;
}
819
#endif /* CONFIG_BPF_JIT */
820

821 822
/* Base function for offset calculation. Needs to go into .text section,
 * therefore keeping it non-static as well; will also be used by JITs
823 824 825
 * anyway later on, so do not let the compiler omit it. This also needs
 * to go into kallsyms for correlation from e.g. bpftool, so naming
 * must not change.
826 827 828 829 830
 */
noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
{
	return 0;
}
831
EXPORT_SYMBOL_GPL(__bpf_call_base);
832

833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
/* All UAPI available opcodes. */
#define BPF_INSN_MAP(INSN_2, INSN_3)		\
	/* 32 bit ALU operations. */		\
	/*   Register based. */			\
	INSN_3(ALU, ADD, X),			\
	INSN_3(ALU, SUB, X),			\
	INSN_3(ALU, AND, X),			\
	INSN_3(ALU, OR,  X),			\
	INSN_3(ALU, LSH, X),			\
	INSN_3(ALU, RSH, X),			\
	INSN_3(ALU, XOR, X),			\
	INSN_3(ALU, MUL, X),			\
	INSN_3(ALU, MOV, X),			\
	INSN_3(ALU, DIV, X),			\
	INSN_3(ALU, MOD, X),			\
	INSN_2(ALU, NEG),			\
	INSN_3(ALU, END, TO_BE),		\
	INSN_3(ALU, END, TO_LE),		\
	/*   Immediate based. */		\
	INSN_3(ALU, ADD, K),			\
	INSN_3(ALU, SUB, K),			\
	INSN_3(ALU, AND, K),			\
	INSN_3(ALU, OR,  K),			\
	INSN_3(ALU, LSH, K),			\
	INSN_3(ALU, RSH, K),			\
	INSN_3(ALU, XOR, K),			\
	INSN_3(ALU, MUL, K),			\
	INSN_3(ALU, MOV, K),			\
	INSN_3(ALU, DIV, K),			\
	INSN_3(ALU, MOD, K),			\
	/* 64 bit ALU operations. */		\
	/*   Register based. */			\
	INSN_3(ALU64, ADD,  X),			\
	INSN_3(ALU64, SUB,  X),			\
	INSN_3(ALU64, AND,  X),			\
	INSN_3(ALU64, OR,   X),			\
	INSN_3(ALU64, LSH,  X),			\
	INSN_3(ALU64, RSH,  X),			\
	INSN_3(ALU64, XOR,  X),			\
	INSN_3(ALU64, MUL,  X),			\
	INSN_3(ALU64, MOV,  X),			\
	INSN_3(ALU64, ARSH, X),			\
	INSN_3(ALU64, DIV,  X),			\
	INSN_3(ALU64, MOD,  X),			\
	INSN_2(ALU64, NEG),			\
	/*   Immediate based. */		\
	INSN_3(ALU64, ADD,  K),			\
	INSN_3(ALU64, SUB,  K),			\
	INSN_3(ALU64, AND,  K),			\
	INSN_3(ALU64, OR,   K),			\
	INSN_3(ALU64, LSH,  K),			\
	INSN_3(ALU64, RSH,  K),			\
	INSN_3(ALU64, XOR,  K),			\
	INSN_3(ALU64, MUL,  K),			\
	INSN_3(ALU64, MOV,  K),			\
	INSN_3(ALU64, ARSH, K),			\
	INSN_3(ALU64, DIV,  K),			\
	INSN_3(ALU64, MOD,  K),			\
	/* Call instruction. */			\
	INSN_2(JMP, CALL),			\
	/* Exit instruction. */			\
	INSN_2(JMP, EXIT),			\
	/* Jump instructions. */		\
	/*   Register based. */			\
	INSN_3(JMP, JEQ,  X),			\
	INSN_3(JMP, JNE,  X),			\
	INSN_3(JMP, JGT,  X),			\
	INSN_3(JMP, JLT,  X),			\
	INSN_3(JMP, JGE,  X),			\
	INSN_3(JMP, JLE,  X),			\
	INSN_3(JMP, JSGT, X),			\
	INSN_3(JMP, JSLT, X),			\
	INSN_3(JMP, JSGE, X),			\
	INSN_3(JMP, JSLE, X),			\
	INSN_3(JMP, JSET, X),			\
	/*   Immediate based. */		\
	INSN_3(JMP, JEQ,  K),			\
	INSN_3(JMP, JNE,  K),			\
	INSN_3(JMP, JGT,  K),			\
	INSN_3(JMP, JLT,  K),			\
	INSN_3(JMP, JGE,  K),			\
	INSN_3(JMP, JLE,  K),			\
	INSN_3(JMP, JSGT, K),			\
	INSN_3(JMP, JSLT, K),			\
	INSN_3(JMP, JSGE, K),			\
	INSN_3(JMP, JSLE, K),			\
	INSN_3(JMP, JSET, K),			\
	INSN_2(JMP, JA),			\
	/* Store instructions. */		\
	/*   Register based. */			\
	INSN_3(STX, MEM,  B),			\
	INSN_3(STX, MEM,  H),			\
	INSN_3(STX, MEM,  W),			\
	INSN_3(STX, MEM,  DW),			\
	INSN_3(STX, XADD, W),			\
	INSN_3(STX, XADD, DW),			\
	/*   Immediate based. */		\
	INSN_3(ST, MEM, B),			\
	INSN_3(ST, MEM, H),			\
	INSN_3(ST, MEM, W),			\
	INSN_3(ST, MEM, DW),			\
	/* Load instructions. */		\
	/*   Register based. */			\
	INSN_3(LDX, MEM, B),			\
	INSN_3(LDX, MEM, H),			\
	INSN_3(LDX, MEM, W),			\
	INSN_3(LDX, MEM, DW),			\
	/*   Immediate based. */		\
941
	INSN_3(LD, IMM, DW)
942 943 944 945 946 947 948 949 950

bool bpf_opcode_in_insntable(u8 code)
{
#define BPF_INSN_2_TBL(x, y)    [BPF_##x | BPF_##y] = true
#define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true
	static const bool public_insntable[256] = {
		[0 ... 255] = false,
		/* Now overwrite non-defaults ... */
		BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL),
951 952 953 954 955 956 957
		/* UAPI exposed, but rewritten opcodes. cBPF carry-over. */
		[BPF_LD | BPF_ABS | BPF_B] = true,
		[BPF_LD | BPF_ABS | BPF_H] = true,
		[BPF_LD | BPF_ABS | BPF_W] = true,
		[BPF_LD | BPF_IND | BPF_B] = true,
		[BPF_LD | BPF_IND | BPF_H] = true,
		[BPF_LD | BPF_IND | BPF_W] = true,
958 959 960 961 962 963
	};
#undef BPF_INSN_3_TBL
#undef BPF_INSN_2_TBL
	return public_insntable[code];
}

964
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
965
/**
966 967 968
 *	__bpf_prog_run - run eBPF program on a given context
 *	@ctx: is the data we are operating on
 *	@insn: is the array of eBPF instructions
969
 *
970
 * Decode and execute eBPF instructions.
971
 */
972
static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack)
973
{
974
	u64 tmp;
975 976
#define BPF_INSN_2_LBL(x, y)    [BPF_##x | BPF_##y] = &&x##_##y
#define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z
977 978 979
	static const void *jumptable[256] = {
		[0 ... 255] = &&default_label,
		/* Now overwrite non-defaults ... */
980 981
		BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL),
		/* Non-UAPI available opcodes. */
982
		[BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS,
983
		[BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL,
984
	};
985 986
#undef BPF_INSN_3_LBL
#undef BPF_INSN_2_LBL
987
	u32 tail_call_cnt = 0;
988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036

#define CONT	 ({ insn++; goto select_insn; })
#define CONT_JMP ({ insn++; goto select_insn; })

select_insn:
	goto *jumptable[insn->code];

	/* ALU */
#define ALU(OPCODE, OP)			\
	ALU64_##OPCODE##_X:		\
		DST = DST OP SRC;	\
		CONT;			\
	ALU_##OPCODE##_X:		\
		DST = (u32) DST OP (u32) SRC;	\
		CONT;			\
	ALU64_##OPCODE##_K:		\
		DST = DST OP IMM;		\
		CONT;			\
	ALU_##OPCODE##_K:		\
		DST = (u32) DST OP (u32) IMM;	\
		CONT;

	ALU(ADD,  +)
	ALU(SUB,  -)
	ALU(AND,  &)
	ALU(OR,   |)
	ALU(LSH, <<)
	ALU(RSH, >>)
	ALU(XOR,  ^)
	ALU(MUL,  *)
#undef ALU
	ALU_NEG:
		DST = (u32) -DST;
		CONT;
	ALU64_NEG:
		DST = -DST;
		CONT;
	ALU_MOV_X:
		DST = (u32) SRC;
		CONT;
	ALU_MOV_K:
		DST = (u32) IMM;
		CONT;
	ALU64_MOV_X:
		DST = SRC;
		CONT;
	ALU64_MOV_K:
		DST = IMM;
		CONT;
1037 1038 1039 1040
	LD_IMM_DW:
		DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32;
		insn++;
		CONT;
1041 1042 1043 1044 1045 1046 1047
	ALU64_ARSH_X:
		(*(s64 *) &DST) >>= SRC;
		CONT;
	ALU64_ARSH_K:
		(*(s64 *) &DST) >>= IMM;
		CONT;
	ALU64_MOD_X:
A
Alexei Starovoitov 已提交
1048 1049
		div64_u64_rem(DST, SRC, &tmp);
		DST = tmp;
1050 1051 1052 1053 1054 1055
		CONT;
	ALU_MOD_X:
		tmp = (u32) DST;
		DST = do_div(tmp, (u32) SRC);
		CONT;
	ALU64_MOD_K:
A
Alexei Starovoitov 已提交
1056 1057
		div64_u64_rem(DST, IMM, &tmp);
		DST = tmp;
1058 1059 1060 1061 1062 1063
		CONT;
	ALU_MOD_K:
		tmp = (u32) DST;
		DST = do_div(tmp, (u32) IMM);
		CONT;
	ALU64_DIV_X:
A
Alexei Starovoitov 已提交
1064
		DST = div64_u64(DST, SRC);
1065 1066 1067 1068 1069 1070 1071
		CONT;
	ALU_DIV_X:
		tmp = (u32) DST;
		do_div(tmp, (u32) SRC);
		DST = (u32) tmp;
		CONT;
	ALU64_DIV_K:
A
Alexei Starovoitov 已提交
1072
		DST = div64_u64(DST, IMM);
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115
		CONT;
	ALU_DIV_K:
		tmp = (u32) DST;
		do_div(tmp, (u32) IMM);
		DST = (u32) tmp;
		CONT;
	ALU_END_TO_BE:
		switch (IMM) {
		case 16:
			DST = (__force u16) cpu_to_be16(DST);
			break;
		case 32:
			DST = (__force u32) cpu_to_be32(DST);
			break;
		case 64:
			DST = (__force u64) cpu_to_be64(DST);
			break;
		}
		CONT;
	ALU_END_TO_LE:
		switch (IMM) {
		case 16:
			DST = (__force u16) cpu_to_le16(DST);
			break;
		case 32:
			DST = (__force u32) cpu_to_le32(DST);
			break;
		case 64:
			DST = (__force u64) cpu_to_le64(DST);
			break;
		}
		CONT;

	/* CALL */
	JMP_CALL:
		/* Function call scratches BPF_R1-BPF_R5 registers,
		 * preserves BPF_R6-BPF_R9, and stores return value
		 * into BPF_R0.
		 */
		BPF_R0 = (__bpf_call_base + insn->imm)(BPF_R1, BPF_R2, BPF_R3,
						       BPF_R4, BPF_R5);
		CONT;

1116 1117 1118 1119 1120 1121 1122
	JMP_CALL_ARGS:
		BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
							    BPF_R3, BPF_R4,
							    BPF_R5,
							    insn + insn->off + 1);
		CONT;

1123 1124 1125 1126
	JMP_TAIL_CALL: {
		struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2;
		struct bpf_array *array = container_of(map, struct bpf_array, map);
		struct bpf_prog *prog;
1127
		u32 index = BPF_R3;
1128 1129 1130 1131 1132 1133 1134 1135

		if (unlikely(index >= array->map.max_entries))
			goto out;
		if (unlikely(tail_call_cnt > MAX_TAIL_CALL_CNT))
			goto out;

		tail_call_cnt++;

1136
		prog = READ_ONCE(array->ptrs[index]);
1137
		if (!prog)
1138 1139
			goto out;

1140 1141 1142 1143 1144
		/* ARG1 at this point is guaranteed to point to CTX from
		 * the verifier side due to the fact that the tail call is
		 * handeled like a helper, that is, bpf_tail_call_proto,
		 * where arg1_type is ARG_PTR_TO_CTX.
		 */
1145 1146 1147 1148 1149
		insn = prog->insnsi;
		goto select_insn;
out:
		CONT;
	}
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
	/* JMP */
	JMP_JA:
		insn += insn->off;
		CONT;
	JMP_JEQ_X:
		if (DST == SRC) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JEQ_K:
		if (DST == IMM) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JNE_X:
		if (DST != SRC) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JNE_K:
		if (DST != IMM) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JGT_X:
		if (DST > SRC) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JGT_K:
		if (DST > IMM) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
	JMP_JLT_X:
		if (DST < SRC) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JLT_K:
		if (DST < IMM) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
	JMP_JGE_X:
		if (DST >= SRC) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JGE_K:
		if (DST >= IMM) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
	JMP_JLE_X:
		if (DST <= SRC) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JLE_K:
		if (DST <= IMM) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
	JMP_JSGT_X:
		if (((s64) DST) > ((s64) SRC)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JSGT_K:
		if (((s64) DST) > ((s64) IMM)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
	JMP_JSLT_X:
		if (((s64) DST) < ((s64) SRC)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JSLT_K:
		if (((s64) DST) < ((s64) IMM)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
	JMP_JSGE_X:
		if (((s64) DST) >= ((s64) SRC)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JSGE_K:
		if (((s64) DST) >= ((s64) IMM)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273
	JMP_JSLE_X:
		if (((s64) DST) <= ((s64) SRC)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JSLE_K:
		if (((s64) DST) <= ((s64) IMM)) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315
	JMP_JSET_X:
		if (DST & SRC) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_JSET_K:
		if (DST & IMM) {
			insn += insn->off;
			CONT_JMP;
		}
		CONT;
	JMP_EXIT:
		return BPF_R0;

	/* STX and ST and LDX*/
#define LDST(SIZEOP, SIZE)						\
	STX_MEM_##SIZEOP:						\
		*(SIZE *)(unsigned long) (DST + insn->off) = SRC;	\
		CONT;							\
	ST_MEM_##SIZEOP:						\
		*(SIZE *)(unsigned long) (DST + insn->off) = IMM;	\
		CONT;							\
	LDX_MEM_##SIZEOP:						\
		DST = *(SIZE *)(unsigned long) (SRC + insn->off);	\
		CONT;

	LDST(B,   u8)
	LDST(H,  u16)
	LDST(W,  u32)
	LDST(DW, u64)
#undef LDST
	STX_XADD_W: /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */
		atomic_add((u32) SRC, (atomic_t *)(unsigned long)
			   (DST + insn->off));
		CONT;
	STX_XADD_DW: /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */
		atomic64_add((u64) SRC, (atomic64_t *)(unsigned long)
			     (DST + insn->off));
		CONT;

	default_label:
1316 1317 1318 1319 1320 1321 1322 1323
		/* If we ever reach this, we have a bug somewhere. Die hard here
		 * instead of just returning 0; we could be somewhere in a subprog,
		 * so execution could continue otherwise which we do /not/ want.
		 *
		 * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable().
		 */
		pr_warn("BPF interpreter: unknown opcode %02x\n", insn->code);
		BUG_ON(1);
1324 1325
		return 0;
}
1326 1327
STACK_FRAME_NON_STANDARD(___bpf_prog_run); /* jump table */

1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
#define PROG_NAME(stack_size) __bpf_prog_run##stack_size
#define DEFINE_BPF_PROG_RUN(stack_size) \
static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \
{ \
	u64 stack[stack_size / sizeof(u64)]; \
	u64 regs[MAX_BPF_REG]; \
\
	FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
	ARG1 = (u64) (unsigned long) ctx; \
	return ___bpf_prog_run(regs, insn, stack); \
1338
}
1339

1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
#define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size
#define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \
static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \
				      const struct bpf_insn *insn) \
{ \
	u64 stack[stack_size / sizeof(u64)]; \
	u64 regs[MAX_BPF_REG]; \
\
	FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
	BPF_R1 = r1; \
	BPF_R2 = r2; \
	BPF_R3 = r3; \
	BPF_R4 = r4; \
	BPF_R5 = r5; \
	return ___bpf_prog_run(regs, insn, stack); \
}

1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367
#define EVAL1(FN, X) FN(X)
#define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y)
#define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y)
#define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y)
#define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y)
#define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y)

EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192);
EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384);
EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512);

1368 1369 1370 1371
EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192);
EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384);
EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512);

1372 1373 1374 1375 1376 1377 1378 1379
#define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size),

static unsigned int (*interpreters[])(const void *ctx,
				      const struct bpf_insn *insn) = {
EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192)
EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384)
EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
};
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
#undef PROG_NAME_LIST
#define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size),
static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
				  const struct bpf_insn *insn) = {
EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192)
EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384)
EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
};
#undef PROG_NAME_LIST

void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
{
	stack_depth = max_t(u32, stack_depth, 1);
	insn->off = (s16) insn->imm;
	insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
		__bpf_call_base_args;
	insn->code = BPF_JMP | BPF_CALL_ARGS;
}
1398

1399
#else
1400 1401
static unsigned int __bpf_prog_ret0_warn(const void *ctx,
					 const struct bpf_insn *insn)
1402
{
1403 1404 1405 1406
	/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON
	 * is not working properly, so warn about it!
	 */
	WARN_ON_ONCE(1);
1407 1408 1409 1410
	return 0;
}
#endif

D
Daniel Borkmann 已提交
1411 1412
bool bpf_prog_array_compatible(struct bpf_array *array,
			       const struct bpf_prog *fp)
1413
{
1414 1415 1416
	if (fp->kprobe_override)
		return false;

D
Daniel Borkmann 已提交
1417 1418 1419 1420
	if (!array->owner_prog_type) {
		/* There's no owner yet where we could check for
		 * compatibility.
		 */
1421 1422
		array->owner_prog_type = fp->type;
		array->owner_jited = fp->jited;
D
Daniel Borkmann 已提交
1423 1424

		return true;
1425
	}
D
Daniel Borkmann 已提交
1426 1427 1428

	return array->owner_prog_type == fp->type &&
	       array->owner_jited == fp->jited;
1429 1430
}

D
Daniel Borkmann 已提交
1431
static int bpf_check_tail_call(const struct bpf_prog *fp)
1432 1433 1434 1435 1436
{
	struct bpf_prog_aux *aux = fp->aux;
	int i;

	for (i = 0; i < aux->used_map_cnt; i++) {
D
Daniel Borkmann 已提交
1437
		struct bpf_map *map = aux->used_maps[i];
1438 1439 1440 1441
		struct bpf_array *array;

		if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
			continue;
D
Daniel Borkmann 已提交
1442

1443 1444 1445 1446 1447 1448 1449 1450
		array = container_of(map, struct bpf_array, map);
		if (!bpf_prog_array_compatible(array, fp))
			return -EINVAL;
	}

	return 0;
}

1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
static void bpf_prog_select_func(struct bpf_prog *fp)
{
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
	u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);

	fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1];
#else
	fp->bpf_func = __bpf_prog_ret0_warn;
#endif
}

1462
/**
D
Daniel Borkmann 已提交
1463
 *	bpf_prog_select_runtime - select exec runtime for BPF program
1464
 *	@fp: bpf_prog populated with internal BPF program
1465
 *	@err: pointer to error variable
1466
 *
D
Daniel Borkmann 已提交
1467 1468
 * Try to JIT eBPF program, if JIT is not available, use interpreter.
 * The BPF program will be executed via BPF_PROG_RUN() macro.
1469
 */
1470
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
1471
{
1472 1473 1474 1475 1476
	/* In case of BPF to BPF calls, verifier did all the prep
	 * work with regards to JITing, etc.
	 */
	if (fp->bpf_func)
		goto finalize;
1477

1478
	bpf_prog_select_func(fp);
1479

1480 1481 1482 1483 1484 1485
	/* eBPF JITs can rewrite the program in case constant
	 * blinding is active. However, in case of error during
	 * blinding, bpf_int_jit_compile() must always return a
	 * valid program, which in this case would simply not
	 * be JITed, but falls back to the interpreter.
	 */
1486 1487
	if (!bpf_prog_is_dev_bound(fp->aux)) {
		fp = bpf_int_jit_compile(fp);
1488 1489 1490 1491 1492 1493
#ifdef CONFIG_BPF_JIT_ALWAYS_ON
		if (!fp->jited) {
			*err = -ENOTSUPP;
			return fp;
		}
#endif
1494 1495 1496 1497 1498
	} else {
		*err = bpf_prog_offload_compile(fp);
		if (*err)
			return fp;
	}
1499 1500

finalize:
1501
	bpf_prog_lock_ro(fp);
1502

D
Daniel Borkmann 已提交
1503 1504 1505 1506 1507
	/* The tail call compatibility check can only be done at
	 * this late stage as we need to determine, if we deal
	 * with JITed or non JITed program concatenations and not
	 * all eBPF JITs might immediately support all features.
	 */
1508
	*err = bpf_check_tail_call(fp);
1509

1510
	return fp;
1511
}
1512
EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
1513

1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
static unsigned int __bpf_prog_ret1(const void *ctx,
				    const struct bpf_insn *insn)
{
	return 1;
}

static struct bpf_prog_dummy {
	struct bpf_prog prog;
} dummy_bpf_prog = {
	.prog = {
		.bpf_func = __bpf_prog_ret1,
	},
};

1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558
/* to avoid allocating empty bpf_prog_array for cgroups that
 * don't have bpf program attached use one global 'empty_prog_array'
 * It will not be modified the caller of bpf_prog_array_alloc()
 * (since caller requested prog_cnt == 0)
 * that pointer should be 'freed' by bpf_prog_array_free()
 */
static struct {
	struct bpf_prog_array hdr;
	struct bpf_prog *null_prog;
} empty_prog_array = {
	.null_prog = NULL,
};

struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
{
	if (prog_cnt)
		return kzalloc(sizeof(struct bpf_prog_array) +
			       sizeof(struct bpf_prog *) * (prog_cnt + 1),
			       flags);

	return &empty_prog_array.hdr;
}

void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
{
	if (!progs ||
	    progs == (struct bpf_prog_array __rcu *)&empty_prog_array.hdr)
		return;
	kfree_rcu(progs, rcu);
}

1559 1560 1561 1562 1563 1564 1565 1566
int bpf_prog_array_length(struct bpf_prog_array __rcu *progs)
{
	struct bpf_prog **prog;
	u32 cnt = 0;

	rcu_read_lock();
	prog = rcu_dereference(progs)->progs;
	for (; *prog; prog++)
1567 1568
		if (*prog != &dummy_bpf_prog.prog)
			cnt++;
1569 1570 1571 1572
	rcu_read_unlock();
	return cnt;
}

1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
static bool bpf_prog_array_copy_core(struct bpf_prog **prog,
				     u32 *prog_ids,
				     u32 request_cnt)
{
	int i = 0;

	for (; *prog; prog++) {
		if (*prog == &dummy_bpf_prog.prog)
			continue;
		prog_ids[i] = (*prog)->aux->id;
		if (++i == request_cnt) {
			prog++;
			break;
		}
	}

	return !!(*prog);
}

1592 1593 1594 1595
int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
				__u32 __user *prog_ids, u32 cnt)
{
	struct bpf_prog **prog;
1596 1597
	unsigned long err = 0;
	bool nospc;
1598
	u32 *ids;
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609

	/* users of this function are doing:
	 * cnt = bpf_prog_array_length();
	 * if (cnt > 0)
	 *     bpf_prog_array_copy_to_user(..., cnt);
	 * so below kcalloc doesn't need extra cnt > 0 check, but
	 * bpf_prog_array_length() releases rcu lock and
	 * prog array could have been swapped with empty or larger array,
	 * so always copy 'cnt' prog_ids to the user.
	 * In a rare race the user will see zero prog_ids
	 */
1610
	ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN);
1611 1612
	if (!ids)
		return -ENOMEM;
1613 1614
	rcu_read_lock();
	prog = rcu_dereference(progs)->progs;
1615
	nospc = bpf_prog_array_copy_core(prog, ids, cnt);
1616
	rcu_read_unlock();
1617 1618 1619 1620 1621
	err = copy_to_user(prog_ids, ids, cnt * sizeof(u32));
	kfree(ids);
	if (err)
		return -EFAULT;
	if (nospc)
1622 1623 1624 1625
		return -ENOSPC;
	return 0;
}

1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs,
				struct bpf_prog *old_prog)
{
	struct bpf_prog **prog = progs->progs;

	for (; *prog; prog++)
		if (*prog == old_prog) {
			WRITE_ONCE(*prog, &dummy_bpf_prog.prog);
			break;
		}
}

int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
			struct bpf_prog *exclude_prog,
			struct bpf_prog *include_prog,
			struct bpf_prog_array **new_array)
{
	int new_prog_cnt, carry_prog_cnt = 0;
	struct bpf_prog **existing_prog;
	struct bpf_prog_array *array;
1646
	bool found_exclude = false;
1647 1648 1649 1650 1651 1652 1653 1654
	int new_prog_idx = 0;

	/* Figure out how many existing progs we need to carry over to
	 * the new array.
	 */
	if (old_array) {
		existing_prog = old_array->progs;
		for (; *existing_prog; existing_prog++) {
1655 1656 1657 1658 1659
			if (*existing_prog == exclude_prog) {
				found_exclude = true;
				continue;
			}
			if (*existing_prog != &dummy_bpf_prog.prog)
1660 1661 1662 1663 1664 1665
				carry_prog_cnt++;
			if (*existing_prog == include_prog)
				return -EEXIST;
		}
	}

1666 1667 1668
	if (exclude_prog && !found_exclude)
		return -ENOENT;

1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
	/* How many progs (not NULL) will be in the new array? */
	new_prog_cnt = carry_prog_cnt;
	if (include_prog)
		new_prog_cnt += 1;

	/* Do we have any prog (not NULL) in the new array? */
	if (!new_prog_cnt) {
		*new_array = NULL;
		return 0;
	}

	/* +1 as the end of prog_array is marked with NULL */
	array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL);
	if (!array)
		return -ENOMEM;

	/* Fill in the new prog array */
	if (carry_prog_cnt) {
		existing_prog = old_array->progs;
		for (; *existing_prog; existing_prog++)
			if (*existing_prog != exclude_prog &&
			    *existing_prog != &dummy_bpf_prog.prog)
				array->progs[new_prog_idx++] = *existing_prog;
	}
	if (include_prog)
		array->progs[new_prog_idx++] = include_prog;
	array->progs[new_prog_idx] = NULL;
	*new_array = array;
	return 0;
}

1700
int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
1701 1702
			     u32 *prog_ids, u32 request_cnt,
			     u32 *prog_cnt)
1703
{
1704
	struct bpf_prog **prog;
1705 1706 1707 1708 1709
	u32 cnt = 0;

	if (array)
		cnt = bpf_prog_array_length(array);

1710
	*prog_cnt = cnt;
1711 1712 1713 1714 1715

	/* return early if user requested only program count or nothing to copy */
	if (!request_cnt || !cnt)
		return 0;

1716 1717 1718 1719
	/* this function is called under trace/bpf_trace.c: bpf_event_mutex */
	prog = rcu_dereference_check(array, 1)->progs;
	return bpf_prog_array_copy_core(prog, prog_ids, request_cnt) ? -ENOSPC
								     : 0;
1720 1721
}

1722 1723
static void bpf_prog_free_deferred(struct work_struct *work)
{
1724
	struct bpf_prog_aux *aux;
1725
	int i;
1726

1727
	aux = container_of(work, struct bpf_prog_aux, work);
1728 1729
	if (bpf_prog_is_dev_bound(aux))
		bpf_prog_offload_destroy(aux->prog);
Y
Yonghong Song 已提交
1730 1731 1732 1733
#ifdef CONFIG_PERF_EVENTS
	if (aux->prog->has_callchain_buf)
		put_callchain_buffers();
#endif
1734 1735 1736 1737 1738 1739 1740 1741
	for (i = 0; i < aux->func_cnt; i++)
		bpf_jit_free(aux->func[i]);
	if (aux->func_cnt) {
		kfree(aux->func);
		bpf_prog_unlock_free(aux->prog);
	} else {
		bpf_jit_free(aux->prog);
	}
1742 1743 1744
}

/* Free internal BPF program */
1745
void bpf_prog_free(struct bpf_prog *fp)
1746
{
1747
	struct bpf_prog_aux *aux = fp->aux;
1748

1749 1750
	INIT_WORK(&aux->work, bpf_prog_free_deferred);
	schedule_work(&aux->work);
1751
}
1752
EXPORT_SYMBOL_GPL(bpf_prog_free);
A
Alexei Starovoitov 已提交
1753

1754 1755 1756 1757 1758 1759 1760 1761
/* RNG for unpriviledged user space with separated state from prandom_u32(). */
static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state);

void bpf_user_rnd_init_once(void)
{
	prandom_init_once(&bpf_user_rnd_state);
}

1762
BPF_CALL_0(bpf_user_rnd_u32)
1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
{
	/* Should someone ever have the rather unwise idea to use some
	 * of the registers passed into this function, then note that
	 * this function is called from native eBPF and classic-to-eBPF
	 * transformations. Register assignments from both sides are
	 * different, f.e. classic always sets fn(ctx, A, X) here.
	 */
	struct rnd_state *state;
	u32 res;

	state = &get_cpu_var(bpf_user_rnd_state);
	res = prandom_u32_state(state);
S
Shaohua Li 已提交
1775
	put_cpu_var(bpf_user_rnd_state);
1776 1777 1778 1779

	return res;
}

1780 1781 1782 1783 1784
/* Weak definitions of helper functions in case we don't have bpf syscall. */
const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;
const struct bpf_func_proto bpf_map_update_elem_proto __weak;
const struct bpf_func_proto bpf_map_delete_elem_proto __weak;

1785
const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;
1786
const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;
1787
const struct bpf_func_proto bpf_get_numa_node_id_proto __weak;
1788
const struct bpf_func_proto bpf_ktime_get_ns_proto __weak;
1789

1790 1791 1792
const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak;
const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak;
const struct bpf_func_proto bpf_get_current_comm_proto __weak;
1793
const struct bpf_func_proto bpf_sock_map_update_proto __weak;
1794
const struct bpf_func_proto bpf_sock_hash_update_proto __weak;
1795
const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak;
1796

1797 1798 1799 1800
const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)
{
	return NULL;
}
1801

1802 1803 1804
u64 __weak
bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
		 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
1805
{
1806
	return -ENOTSUPP;
1807
}
J
Jakub Kicinski 已提交
1808
EXPORT_SYMBOL_GPL(bpf_event_output);
1809

D
Daniel Borkmann 已提交
1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
/* Always built-in helper functions. */
const struct bpf_func_proto bpf_tail_call_proto = {
	.func		= NULL,
	.gpl_only	= false,
	.ret_type	= RET_VOID,
	.arg1_type	= ARG_PTR_TO_CTX,
	.arg2_type	= ARG_CONST_MAP_PTR,
	.arg3_type	= ARG_ANYTHING,
};

1820 1821 1822 1823
/* Stub for JITs that only support cBPF. eBPF programs are interpreted.
 * It is encouraged to implement bpf_int_jit_compile() instead, so that
 * eBPF and implicitly also cBPF can get JITed!
 */
1824
struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
D
Daniel Borkmann 已提交
1825
{
1826
	return prog;
D
Daniel Borkmann 已提交
1827 1828
}

1829 1830 1831 1832 1833 1834 1835
/* Stub for JITs that support eBPF. All cBPF code gets transformed into
 * eBPF by the kernel and is later compiled by bpf_int_jit_compile().
 */
void __weak bpf_jit_compile(struct bpf_prog *prog)
{
}

1836
bool __weak bpf_helper_changes_pkt_data(void *func)
A
Alexei Starovoitov 已提交
1837 1838 1839 1840
{
	return false;
}

A
Alexei Starovoitov 已提交
1841 1842 1843 1844 1845 1846 1847 1848
/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
 * skb_copy_bits(), so provide a weak definition of it for NET-less config.
 */
int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to,
			 int len)
{
	return -EFAULT;
}
1849 1850 1851 1852 1853 1854

/* All definitions of tracepoints related to BPF. */
#define CREATE_TRACE_POINTS
#include <linux/bpf_trace.h>

EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception);