Skip to content

K
Kernel

openeuler / Kernel
大约 2 年 前同步成功

通知 8
Star 0
Fork 0
K
Kernel
提交 f77b8846 编写于 作者: M Mao Minkai 提交者： guzitao
浏览文件
操作

sw64: bpf: fix ebpf jit compiler 

Sunway inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I5PNGJ

--------------------------------

This patch makes following changes to ebpf jit compiler:
   * implement proper XADD instructions
   * implement 32-bit ARSH instructions
   * implement DIV and MOD instructions using helper functions
   * reorganize header file to make it easier to read
   * optimize load immediate helper functions
   * make sure ILLEGAL_INSN will throw instruction fault
   * make sure fields in jited instrctions won't overflow
   * restore GP register when exit
   * make sure 32-bit alu functions are unsigned
   * make sure 32-bit results are zero extended to 64 bits
   * make sure function addr are stored in $27 so callee can calculate
     GP correctly
   * track free temporary registers to make sure we won't accidentally
     clobber useful data
   * fix register mapping
   * fix host to be algorithm
   * fix offset calculation of branch instructions
   * fix tail call

Result of "test_bpf.ko": 378 PASSED, 0 FAILED, [366/366 JIT'ed]
Signed-off-by: NMao Minkai <maominkai@wxiat.com>
Signed-off-by: NGu Zitao <guzitao@wxiat.com>
上级 bd312722
隐藏空白更改
内联 并排
Showing with 871 addition and 433 deletion
arch/sw_64/net/bpf_jit.h
浏览文件 @ f77b8846
...@@ -21,80 +21,82 @@ ...@@ -21,80 +21,82 @@
#ifndef _SW64_BPF_JIT_H #ifndef _SW64_BPF_JIT_H
#define _SW64_BPF_JIT_H #define _SW64_BPF_JIT_H
/* SW64 instruction field shift */
#define SW64_BPF_OPCODE_OFFSET 26 #define SW64_BPF_OPCODE_OFFSET 26
#define SW64_BPF_RA_OFFSET 21 #define SW64_BPF_RA_OFFSET 21
#define SW64_BPF_RB_OFFSET 16 #define SW64_BPF_RB_OFFSET 16
#define SW64_BPF_SIMPLE_ALU_IMM_OFFSET 13 #define SW64_BPF_SIMPLE_ALU_IMM_OFFSET 13
#define SW64_BPF_SIMPLE_ALU_FUNC_OFFSET 5 #define SW64_BPF_SIMPLE_ALU_FUNC_OFFSET 5
#define SW64_BPF_SIMPLE_ALU_RC_OFFSET 0 #define SW64_BPF_SIMPLE_ALU_RC_OFFSET 0
#define SW64_BPF_LS_FUNC_OFFSET 12
#define SW64_BPF_OPCODE_BR_CALL 0x01 /* SW64 instruction opcodes */
#define SW64_BPF_OPCODE_BR_RET 0x02 #define SW64_BPF_OPCODE_CALL 0x01
#define SW64_BPF_OPCODE_BR_JMP 0x03 #define SW64_BPF_OPCODE_RET 0x02
#define SW64_BPF_OPCODE_BR_BR 0x04 #define SW64_BPF_OPCODE_JMP 0x03
#define SW64_BPF_OPCODE_BR_BSR 0x05 #define SW64_BPF_OPCODE_BR 0x04
#define SW64_BPF_OPCODE_BR_BEQ 0x30 #define SW64_BPF_OPCODE_BSR 0x05
#define SW64_BPF_OPCODE_BR_BNE 0x31 #define SW64_BPF_OPCODE_MISC 0x06
#define SW64_BPF_OPCODE_BR_BLT 0x32 #define SW64_BPF_OPCODE_LOCK 0x08
#define SW64_BPF_OPCODE_BR_BLE 0x33
#define SW64_BPF_OPCODE_BR_BGT 0x34
#define SW64_BPF_OPCODE_BR_BGE 0x35
#define SW64_BPF_OPCODE_BR_BLBC 0x36
#define SW64_BPF_OPCODE_BR_BLBS 0x37
#define SW64_BPF_OPCODE_LS_LDBU 0x20
#define SW64_BPF_OPCODE_LS_LDHU 0x21
#define SW64_BPF_OPCODE_LS_LDW 0x22
#define SW64_BPF_OPCODE_LS_LDL 0x23
#define SW64_BPF_OPCODE_LS_STB 0x28
#define SW64_BPF_OPCODE_LS_STH 0x29
#define SW64_BPF_OPCODE_LS_STW 0x2A
#define SW64_BPF_OPCODE_LS_STL 0x2B
#define SW64_BPF_OPCODE_LS_LDI 0x3E
#define SW64_BPF_OPCODE_LS_LDIH 0x3F
#define SW64_BPF_OPCODE_ALU_REG 0x10 #define SW64_BPF_OPCODE_ALU_REG 0x10
#define SW64_BPF_OPCODE_ALU_IMM 0x12 #define SW64_BPF_OPCODE_ALU_IMM 0x12
#define SW64_BPF_OPCODE_LDBU 0x20
#define SW64_BPF_OPCODE_LDHU 0x21
#define SW64_BPF_OPCODE_LDW 0x22
#define SW64_BPF_OPCODE_LDL 0x23
#define SW64_BPF_OPCODE_STB 0x28
#define SW64_BPF_OPCODE_STH 0x29
#define SW64_BPF_OPCODE_STW 0x2A
#define SW64_BPF_OPCODE_STL 0x2B
#define SW64_BPF_OPCODE_BEQ 0x30
#define SW64_BPF_OPCODE_BNE 0x31
#define SW64_BPF_OPCODE_BLT 0x32
#define SW64_BPF_OPCODE_BLE 0x33
#define SW64_BPF_OPCODE_BGT 0x34
#define SW64_BPF_OPCODE_BGE 0x35
#define SW64_BPF_OPCODE_BLBC 0x36
#define SW64_BPF_OPCODE_BLBS 0x37
#define SW64_BPF_OPCODE_LDI 0x3E
#define SW64_BPF_OPCODE_LDIH 0x3F
/* SW64 MISC instructions function codes */
#define SW64_BPF_FUNC_MISC_RD_F 0x1000
#define SW64_BPF_FUNC_MISC_WR_F 0x1020
/* SW64 LOCK instructions function codes */
#define SW64_BPF_FUNC_LOCK_LLDW 0x0
#define SW64_BPF_FUNC_LOCK_LLDL 0x1
#define SW64_BPF_FUNC_LOCK_LSTW 0x8
#define SW64_BPF_FUNC_LOCK_LSTL 0x9
/* SW64 ALU instructions function codes */
#define SW64_BPF_FUNC_ALU_ADDW 0x00 #define SW64_BPF_FUNC_ALU_ADDW 0x00
#define SW64_BPF_FUNC_ALU_SUBW 0x01 #define SW64_BPF_FUNC_ALU_SUBW 0x01
#define SW64_BPF_FUNC_ALU_ADDL 0x08 #define SW64_BPF_FUNC_ALU_ADDL 0x08
#define SW64_BPF_FUNC_ALU_SUBL 0x09 #define SW64_BPF_FUNC_ALU_SUBL 0x09
#define SW64_BPF_FUNC_ALU_MULW 0x10 #define SW64_BPF_FUNC_ALU_MULW 0x10
#define SW64_BPF_FUNC_ALU_MULL 0x18 #define SW64_BPF_FUNC_ALU_MULL 0x18
#define SW64_BPF_FUNC_ALU_CMPEQ 0x28
#define SW64_BPF_FUNC_ALU_CMPLT 0x29
#define SW64_BPF_FUNC_ALU_CMPLE 0x2A
#define SW64_BPF_FUNC_ALU_CMPULT 0x2B
#define SW64_BPF_FUNC_ALU_CMPULE 0x2C
#define SW64_BPF_FUNC_ALU_AND 0x38
#define SW64_BPF_FUNC_ALU_BIC 0x39
#define SW64_BPF_FUNC_ALU_BIS 0x3A
#define SW64_BPF_FUNC_ALU_ORNOT 0x3B
#define SW64_BPF_FUNC_ALU_XOR 0x3C
#define SW64_BPF_FUNC_ALU_EQV 0x3D
#define SW64_BPF_FUNC_ALU_SLL 0x48
#define SW64_BPF_FUNC_ALU_SRL 0x49
#define SW64_BPF_FUNC_ALU_SRA 0x4A
#define SW64_BPF_FUNC_ALU_ZAP 0x68 #define SW64_BPF_FUNC_ALU_ZAP 0x68
#define SW64_BPF_FUNC_ALU_ZAPNOT 0x69 #define SW64_BPF_FUNC_ALU_ZAPNOT 0x69
#define SW64_BPF_FUNC_ALU_SEXTB 0x6A #define SW64_BPF_FUNC_ALU_SEXTB 0x6A
#define SW64_BPF_FUNC_ALU_SEXTH 0x6B #define SW64_BPF_FUNC_ALU_SEXTH 0x6B
#define SW64_BPF_OPCODE_BS_REG 0x10
#define SW64_BPF_OPCODE_BS_IMM 0x12
#define SW64_BPF_FUNC_BS_SLL 0x48
#define SW64_BPF_FUNC_BS_SRL 0x49
#define SW64_BPF_FUNC_BS_SRA 0x4A
#define SW64_BPF_OPCODE_LOGIC_REG 0x10
#define SW64_BPF_OPCODE_LOGIC_IMM 0x12
#define SW64_BPF_FUNC_LOGIC_AND 0x38
#define SW64_BPF_FUNC_LOGIC_BIC 0x39
#define SW64_BPF_FUNC_LOGIC_BIS 0x3A
#define SW64_BPF_FUNC_LOGIC_ORNOT 0x3B
#define SW64_BPF_FUNC_LOGIC_XOR 0x3C
#define SW64_BPF_FUNC_LOGIC_EQV 0x3D
#define SW64_BPF_OPCODE_CMP_REG 0x10
#define SW64_BPF_OPCODE_CMP_IMM 0x12
#define SW64_BPF_FUNC_CMP_EQ 0x28
#define SW64_BPF_FUNC_CMP_LT 0x29
#define SW64_BPF_FUNC_CMP_LE 0x2A
#define SW64_BPF_FUNC_CMP_ULT 0x2B
#define SW64_BPF_FUNC_CMP_ULE 0x2C
/* special instuction used in jit_fill_hole() */ /* special instuction used in jit_fill_hole() */
#define SW64_BPF_ILLEGAL_INSN ((1 << 25) | 0x80) #define SW64_BPF_ILLEGAL_INSN (0x1bff1000) /* rd_f $31 */
enum sw64_bpf_registers { enum sw64_bpf_registers {
SW64_BPF_REG_V0 = 0, /* keep return value */ SW64_BPF_REG_V0 = 0, /* keep return value */
...@@ -135,25 +137,45 @@ enum sw64_bpf_registers { ...@@ -135,25 +137,45 @@ enum sw64_bpf_registers {
/* SW64 load and store instructions */ /* SW64 load and store instructions */
#define SW64_BPF_LDBU(dst, rb, offset16) \ #define SW64_BPF_LDBU(dst, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_LDBU, dst, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDBU, dst, rb, offset16)
#define SW64_BPF_LDHU(dst, rb, offset16) \ #define SW64_BPF_LDHU(dst, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_LDHU, dst, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDHU, dst, rb, offset16)
#define SW64_BPF_LDW(dst, rb, offset16) \ #define SW64_BPF_LDW(dst, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_LDW, dst, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDW, dst, rb, offset16)
#define SW64_BPF_LDL(dst, rb, offset16) \ #define SW64_BPF_LDL(dst, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_LDL, dst, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDL, dst, rb, offset16)
#define SW64_BPF_STB(src, rb, offset16) \ #define SW64_BPF_STB(src, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_STB, src, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STB, src, rb, offset16)
#define SW64_BPF_STH(src, rb, offset16) \ #define SW64_BPF_STH(src, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_STH, src, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STH, src, rb, offset16)
#define SW64_BPF_STW(src, rb, offset16) \ #define SW64_BPF_STW(src, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_STW, src, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STW, src, rb, offset16)
#define SW64_BPF_STL(src, rb, offset16) \ #define SW64_BPF_STL(src, rb, offset16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_STL, src, rb, offset16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STL, src, rb, offset16)
#define SW64_BPF_LDI(dst, rb, imm16) \ #define SW64_BPF_LDI(dst, rb, imm16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_LDI, dst, rb, imm16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDI, dst, rb, imm16)
#define SW64_BPF_LDIH(dst, rb, imm16) \ #define SW64_BPF_LDIH(dst, rb, imm16) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LS_LDIH, dst, rb, imm16) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDIH, dst, rb, imm16)
/* SW64 lock instructions */
#define SW64_BPF_LLDW(ra, rb, offset16) \
sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \
ra, rb, offset16, SW64_BPF_FUNC_LOCK_LLDW)
#define SW64_BPF_LLDL(ra, rb, offset16) \
sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \
ra, rb, offset16, SW64_BPF_FUNC_LOCK_LLDL)
#define SW64_BPF_LSTW(ra, rb, offset16) \
sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \
ra, rb, offset16, SW64_BPF_FUNC_LOCK_LSTW)
#define SW64_BPF_LSTL(ra, rb, offset16) \
sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \
ra, rb, offset16, SW64_BPF_FUNC_LOCK_LSTL)
#define SW64_BPF_RD_F(ra) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_MISC, \
ra, SW64_BPF_REG_ZR, SW64_BPF_FUNC_MISC_RD_F)
#define SW64_BPF_WR_F(ra) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_MISC, \
ra, SW64_BPF_REG_ZR, SW64_BPF_FUNC_MISC_WR_F)
/* SW64 ALU instructions REG format */ /* SW64 ALU instructions REG format */
#define SW64_BPF_ADDW_REG(ra, rb, dst) \ #define SW64_BPF_ADDW_REG(ra, rb, dst) \
...@@ -182,10 +204,10 @@ enum sw64_bpf_registers { ...@@ -182,10 +204,10 @@ enum sw64_bpf_registers {
ra, rb, dst, SW64_BPF_FUNC_ALU_ZAPNOT) ra, rb, dst, SW64_BPF_FUNC_ALU_ZAPNOT)
#define SW64_BPF_SEXTB_REG(rb, dst) \ #define SW64_BPF_SEXTB_REG(rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
0, rb, dst, SW64_BPF_FUNC_ALU_SEXTB) SW64_BPF_REG_ZR, rb, dst, SW64_BPF_FUNC_ALU_SEXTB)
#define SW64_BPF_SEXTH_REG(rb, dst) \ #define SW64_BPF_SEXTH_REG(rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
0, rb, dst, SW64_BPF_FUNC_ALU_SEXTH) SW64_BPF_REG_ZR, rb, dst, SW64_BPF_FUNC_ALU_SEXTH)
/* SW64 ALU instructions IMM format */ /* SW64 ALU instructions IMM format */
#define SW64_BPF_ADDW_IMM(ra, imm8, dst) \ #define SW64_BPF_ADDW_IMM(ra, imm8, dst) \
...@@ -214,130 +236,133 @@ enum sw64_bpf_registers { ...@@ -214,130 +236,133 @@ enum sw64_bpf_registers {
ra, imm8, dst, SW64_BPF_FUNC_ALU_ZAPNOT) ra, imm8, dst, SW64_BPF_FUNC_ALU_ZAPNOT)
#define SW64_BPF_SEXTB_IMM(imm8, dst) \ #define SW64_BPF_SEXTB_IMM(imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
0, imm8, dst, SW64_BPF_FUNC_ALU_SEXTB) SW64_BPF_REG_ZR, imm8, dst, SW64_BPF_FUNC_ALU_SEXTB)
#define SW64_BPF_SEXTH_IMM(imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
SW64_BPF_REG_ZR, imm8, dst, SW64_BPF_FUNC_ALU_SEXTH)
/* SW64 bit shift instructions REG format */ /* SW64 bit shift instructions REG format */
#define SW64_BPF_SLL_REG(src, rb, dst) \ #define SW64_BPF_SLL_REG(src, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_BS_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
src, rb, dst, SW64_BPF_FUNC_BS_SLL) src, rb, dst, SW64_BPF_FUNC_ALU_SLL)
#define SW64_BPF_SRL_REG(src, rb, dst) \ #define SW64_BPF_SRL_REG(src, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_BS_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
src, rb, dst, SW64_BPF_FUNC_BS_SRL) src, rb, dst, SW64_BPF_FUNC_ALU_SRL)
#define SW64_BPF_SRA_REG(src, rb, dst) \ #define SW64_BPF_SRA_REG(src, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_BS_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
src, rb, dst, SW64_BPF_FUNC_BS_SRA) src, rb, dst, SW64_BPF_FUNC_ALU_SRA)
/* SW64 bit shift instructions IMM format */ /* SW64 bit shift instructions IMM format */
#define SW64_BPF_SLL_IMM(src, imm8, dst) \ #define SW64_BPF_SLL_IMM(src, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_BS_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
src, imm8, dst, SW64_BPF_FUNC_BS_SLL) src, imm8, dst, SW64_BPF_FUNC_ALU_SLL)
#define SW64_BPF_SRL_IMM(src, imm8, dst) \ #define SW64_BPF_SRL_IMM(src, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_BS_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
src, imm8, dst, SW64_BPF_FUNC_BS_SRL) src, imm8, dst, SW64_BPF_FUNC_ALU_SRL)
#define SW64_BPF_SRA_IMM(src, imm8, dst) \ #define SW64_BPF_SRA_IMM(src, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_BS_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
src, imm8, dst, SW64_BPF_FUNC_BS_SRA) src, imm8, dst, SW64_BPF_FUNC_ALU_SRA)
/* SW64 control instructions */ /* SW64 control instructions */
#define SW64_BPF_CALL(ra, rb) \ #define SW64_BPF_CALL(ra, rb) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_BR_CALL, ra, rb, 0) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_CALL, ra, rb, 0)
#define SW64_BPF_RET(rb) \ #define SW64_BPF_RET(rb) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_BR_RET, SW64_BPF_REG_ZR, rb, 0) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_RET, SW64_BPF_REG_ZR, rb, 0)
#define SW64_BPF_JMP(ra, rb) \ #define SW64_BPF_JMP(ra, rb) \
sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_BR_JMP, ra, rb, 0) sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_JMP, ra, rb, 0)
#define SW64_BPF_BR(ra, offset) \ #define SW64_BPF_BR(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BR, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR, ra, offset)
#define SW64_BPF_BSR(ra, offset) \ #define SW64_BPF_BSR(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BSR, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BSR, ra, offset)
#define SW64_BPF_BEQ(ra, offset) \ #define SW64_BPF_BEQ(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BEQ, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BEQ, ra, offset)
#define SW64_BPF_BNE(ra, offset) \ #define SW64_BPF_BNE(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BNE, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BNE, ra, offset)
#define SW64_BPF_BLT(ra, offset) \ #define SW64_BPF_BLT(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BLT, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLT, ra, offset)
#define SW64_BPF_BLE(ra, offset) \ #define SW64_BPF_BLE(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BLE, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLE, ra, offset)
#define SW64_BPF_BGT(ra, offset) \ #define SW64_BPF_BGT(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BGT, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BGT, ra, offset)
#define SW64_BPF_BGE(ra, offset) \ #define SW64_BPF_BGE(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BGE, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BGE, ra, offset)
#define SW64_BPF_BLBC(ra, offset) \ #define SW64_BPF_BLBC(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BLBC, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLBC, ra, offset)
#define SW64_BPF_BLBS(ra, offset) \ #define SW64_BPF_BLBS(ra, offset) \
sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR_BLBS, ra, offset) sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLBS, ra, offset)
/* SW64 bit logic instructions REG format */ /* SW64 bit logic instructions REG format */
#define SW64_BPF_AND_REG(ra, rb, dst) \ #define SW64_BPF_AND_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_LOGIC_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_LOGIC_AND) ra, rb, dst, SW64_BPF_FUNC_ALU_AND)
#define SW64_BPF_ANDNOT_REG(ra, rb, dst) \ #define SW64_BPF_ANDNOT_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_LOGIC_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_LOGIC_BIC) ra, rb, dst, SW64_BPF_FUNC_ALU_BIC)
#define SW64_BPF_OR_REG(ra, rb, dst) \ #define SW64_BPF_BIS_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_LOGIC_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_LOGIC_BIS) ra, rb, dst, SW64_BPF_FUNC_ALU_BIS)
#define SW64_BPF_ORNOT_REG(ra, rb, dst) \ #define SW64_BPF_ORNOT_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_LOGIC_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_LOGIC_ORNOT) ra, rb, dst, SW64_BPF_FUNC_ALU_ORNOT)
#define SW64_BPF_XOR_REG(ra, rb, dst) \ #define SW64_BPF_XOR_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_LOGIC_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_LOGIC_XOR) ra, rb, dst, SW64_BPF_FUNC_ALU_XOR)
#define SW64_BPF_EQV_REG(ra, rb, dst) \ #define SW64_BPF_EQV_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_LOGIC_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_LOGIC_EQV) ra, rb, dst, SW64_BPF_FUNC_ALU_EQV)
/* SW64 bit logic instructions IMM format */ /* SW64 bit logic instructions IMM format */
#define SW64_BPF_AND_IMM(ra, imm8, dst) \ #define SW64_BPF_AND_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_LOGIC_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_LOGIC_AND) ra, imm8, dst, SW64_BPF_FUNC_ALU_AND)
#define SW64_BPF_ANDNOT_IMM(ra, imm8, dst) \ #define SW64_BPF_ANDNOT_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_LOGIC_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_LOGIC_BIC) ra, imm8, dst, SW64_BPF_FUNC_ALU_BIC)
#define SW64_BPF_OR_IMM(ra, imm8, dst) \ #define SW64_BPF_BIS_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_LOGIC_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_LOGIC_BIS) ra, imm8, dst, SW64_BPF_FUNC_ALU_BIS)
#define SW64_BPF_ORNOT_IMM(ra, imm8, dst) \ #define SW64_BPF_ORNOT_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_LOGIC_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_LOGIC_ORNOT) ra, imm8, dst, SW64_BPF_FUNC_ALU_ORNOT)
#define SW64_BPF_XOR_IMM(ra, imm8, dst) \ #define SW64_BPF_XOR_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_LOGIC_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_LOGIC_XOR) ra, imm8, dst, SW64_BPF_FUNC_ALU_XOR)
#define SW64_BPF_EQV_IMM(ra, imm8, dst) \ #define SW64_BPF_EQV_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_LOGIC_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_LOGIC_EQV) ra, imm8, dst, SW64_BPF_FUNC_ALU_EQV)
/* SW64 compare instructions REG format */ /* SW64 compare instructions REG format */
#define SW64_BPF_CMPEQ_REG(ra, rb, dst) \ #define SW64_BPF_CMPEQ_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_CMP_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_CMP_EQ) ra, rb, dst, SW64_BPF_FUNC_ALU_CMPEQ)
#define SW64_BPF_CMPLT_REG(ra, rb, dst) \ #define SW64_BPF_CMPLT_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_CMP_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_CMP_LT) ra, rb, dst, SW64_BPF_FUNC_ALU_CMPLT)
#define SW64_BPF_CMPLE_REG(ra, rb, dst) \ #define SW64_BPF_CMPLE_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_CMP_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_CMP_LE) ra, rb, dst, SW64_BPF_FUNC_ALU_CMPLE)
#define SW64_BPF_CMPULT_REG(ra, rb, dst) \ #define SW64_BPF_CMPULT_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_CMP_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_CMP_ULT) ra, rb, dst, SW64_BPF_FUNC_ALU_CMPULT)
#define SW64_BPF_CMPULE_REG(ra, rb, dst) \ #define SW64_BPF_CMPULE_REG(ra, rb, dst) \
sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_CMP_REG, \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \
ra, rb, dst, SW64_BPF_FUNC_CMP_ULE) ra, rb, dst, SW64_BPF_FUNC_ALU_CMPULE)
/* SW64 compare instructions imm format */ /* SW64 compare instructions imm format */
#define SW64_BPF_CMPEQ_IMM(ra, imm8, dst) \ #define SW64_BPF_CMPEQ_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_CMP_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_CMP_EQ) ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPEQ)
#define SW64_BPF_CMPLT_IMM(ra, imm8, dst) \ #define SW64_BPF_CMPLT_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_CMP_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_CMP_LT) ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPLT)
#define SW64_BPF_CMPLE_IMM(ra, imm8, dst) \ #define SW64_BPF_CMPLE_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_CMP_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_CMP_LE) ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPLE)
#define SW64_BPF_CMPULT_IMM(ra, imm8, dst) \ #define SW64_BPF_CMPULT_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_CMP_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_CMP_ULT) ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPULT)
#define SW64_BPF_CMPULE_IMM(ra, imm8, dst) \ #define SW64_BPF_CMPULE_IMM(ra, imm8, dst) \
sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_CMP_IMM, \ sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \
ra, imm8, dst, SW64_BPF_FUNC_CMP_ULE) ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPULE)
#endif /* _SW64_BPF_JIT_H */ #endif /* _SW64_BPF_JIT_H */
arch/sw_64/net/bpf_jit_comp.c
浏览文件 @ f77b8846
...@@ -29,44 +29,34 @@ ...@@ -29,44 +29,34 @@
#include "bpf_jit.h" #include "bpf_jit.h"
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TCALL_CNT (MAX_BPF_JIT_REG + 0)
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
#define TCALL_CNT (MAX_BPF_JIT_REG + 2)
/*
* TO-DO List:
* DIV
* MOD
*/
static const int bpf2sw64[] = { static const int bpf2sw64[] = {
/* return value from in-kernel function, and exit value from eBPF */ /* return value from in-kernel function, and exit value from eBPF */
[BPF_REG_0] = SW64_BPF_REG_V0, [BPF_REG_0] = SW64_BPF_REG_V0,
/* arguments from eBPF program to in-kernel function */ /* arguments from eBPF program to in-kernel function */
[BPF_REG_1] = SW64_BPF_REG_A1, [BPF_REG_1] = SW64_BPF_REG_A0,
[BPF_REG_2] = SW64_BPF_REG_A2, [BPF_REG_2] = SW64_BPF_REG_A1,
[BPF_REG_3] = SW64_BPF_REG_A3, [BPF_REG_3] = SW64_BPF_REG_A2,
[BPF_REG_4] = SW64_BPF_REG_A4, [BPF_REG_4] = SW64_BPF_REG_A3,
[BPF_REG_5] = SW64_BPF_REG_A5, [BPF_REG_5] = SW64_BPF_REG_A4,
/* callee saved registers that in-kernel function will preserve */ /* callee saved registers that in-kernel function will preserve */
[BPF_REG_6] = SW64_BPF_REG_S1, [BPF_REG_6] = SW64_BPF_REG_S0,
[BPF_REG_7] = SW64_BPF_REG_S2, [BPF_REG_7] = SW64_BPF_REG_S1,
[BPF_REG_8] = SW64_BPF_REG_S3, [BPF_REG_8] = SW64_BPF_REG_S2,
[BPF_REG_9] = SW64_BPF_REG_S4, [BPF_REG_9] = SW64_BPF_REG_S3,
/* read-only frame pointer to access stack */ /* read-only frame pointer to access stack */
[BPF_REG_FP] = SW64_BPF_REG_S0, [BPF_REG_FP] = SW64_BPF_REG_FP,
/* temporary registers for internal BPF JIT */
[TMP_REG_1] = SW64_BPF_REG_T1,
[TMP_REG_2] = SW64_BPF_REG_T2,
/* tail_call_cnt */ /* tail_call_cnt */
[TCALL_CNT] = SW64_BPF_REG_S5, [TCALL_CNT] = SW64_BPF_REG_S4,
/* temporary register for blinding constants */ /* temporary register for blinding constants */
[BPF_REG_AX] = SW64_BPF_REG_T12, [BPF_REG_AX] = SW64_BPF_REG_T11,
}; };
struct jit_ctx { struct jit_ctx {
const struct bpf_prog *prog; const struct bpf_prog *prog;
int idx; // JITed instruction index int idx; // JITed instruction index
int current_tmp_reg;
int epilogue_offset; int epilogue_offset;
int *insn_offset; // [bpf_insn_idx] = jited_insn_idx int *insn_offset; // [bpf_insn_idx] = jited_insn_idx
u32 *image; // JITed instruction u32 *image; // JITed instruction
...@@ -83,7 +73,7 @@ static inline u32 sw64_bpf_gen_format_br(int opcode, enum sw64_bpf_registers ra, ...@@ -83,7 +73,7 @@ static inline u32 sw64_bpf_gen_format_br(int opcode, enum sw64_bpf_registers ra,
{ {
opcode = opcode << SW64_BPF_OPCODE_OFFSET; opcode = opcode << SW64_BPF_OPCODE_OFFSET;
ra = ra << SW64_BPF_RA_OFFSET; ra = ra << SW64_BPF_RA_OFFSET;
return opcode | ra | disp; return opcode | ra | (disp & 0x1fffff);
} }
static inline u32 sw64_bpf_gen_format_ls(int opcode, enum sw64_bpf_registers ra, static inline u32 sw64_bpf_gen_format_ls(int opcode, enum sw64_bpf_registers ra,
...@@ -92,7 +82,17 @@ static inline u32 sw64_bpf_gen_format_ls(int opcode, enum sw64_bpf_registers ra, ...@@ -92,7 +82,17 @@ static inline u32 sw64_bpf_gen_format_ls(int opcode, enum sw64_bpf_registers ra,
opcode = opcode << SW64_BPF_OPCODE_OFFSET; opcode = opcode << SW64_BPF_OPCODE_OFFSET;
ra = ra << SW64_BPF_RA_OFFSET; ra = ra << SW64_BPF_RA_OFFSET;
rb = rb << SW64_BPF_RB_OFFSET; rb = rb << SW64_BPF_RB_OFFSET;
return opcode | ra | rb | disp; return opcode | ra | rb | (disp & 0xffff);
}
static inline u32 sw64_bpf_gen_format_ls_func(int opcode, enum sw64_bpf_registers ra,
enum sw64_bpf_registers rb, u16 disp, int function)
{
opcode = opcode << SW64_BPF_OPCODE_OFFSET;
ra = ra << SW64_BPF_RA_OFFSET;
rb = rb << SW64_BPF_RB_OFFSET;
function = function << SW64_BPF_LS_FUNC_OFFSET;
return opcode | ra | rb | function | (disp & 0xfff);
} }
static inline u32 sw64_bpf_gen_format_simple_alu_reg(int opcode, enum sw64_bpf_registers ra, static inline u32 sw64_bpf_gen_format_simple_alu_reg(int opcode, enum sw64_bpf_registers ra,
...@@ -107,12 +107,12 @@ static inline u32 sw64_bpf_gen_format_simple_alu_reg(int opcode, enum sw64_bpf_r ...@@ -107,12 +107,12 @@ static inline u32 sw64_bpf_gen_format_simple_alu_reg(int opcode, enum sw64_bpf_r
} }
static inline u32 sw64_bpf_gen_format_simple_alu_imm(int opcode, enum sw64_bpf_registers ra, static inline u32 sw64_bpf_gen_format_simple_alu_imm(int opcode, enum sw64_bpf_registers ra,
enum sw64_bpf_registers rc, u8 imm, int function) u32 imm, enum sw64_bpf_registers rc, int function)
{ {
opcode = opcode << SW64_BPF_OPCODE_OFFSET; opcode = opcode << SW64_BPF_OPCODE_OFFSET;
ra = ra << SW64_BPF_RA_OFFSET; ra = ra << SW64_BPF_RA_OFFSET;
imm = (imm & 0xff) << SW64_BPF_SIMPLE_ALU_IMM_OFFSET;
rc = rc << SW64_BPF_SIMPLE_ALU_RC_OFFSET; rc = rc << SW64_BPF_SIMPLE_ALU_RC_OFFSET;
imm = imm << SW64_BPF_SIMPLE_ALU_IMM_OFFSET;
function = function << SW64_BPF_SIMPLE_ALU_FUNC_OFFSET; function = function << SW64_BPF_SIMPLE_ALU_FUNC_OFFSET;
return opcode | ra | imm | function | rc; return opcode | ra | imm | function | rc;
} }
...@@ -125,57 +125,85 @@ static inline void emit(const u32 insn, struct jit_ctx *ctx) ...@@ -125,57 +125,85 @@ static inline void emit(const u32 insn, struct jit_ctx *ctx)
ctx->idx++; ctx->idx++;
} }
static inline void emit_sw64_ldu64(const int dst, const u64 imm64, struct jit_ctx *ctx) static inline int get_tmp_reg(struct jit_ctx *ctx)
{ {
u16 imm_tmp; ctx->current_tmp_reg++;
int reg_tmp = SW64_BPF_REG_T8; /* Do not use 22-25. Should be more than enough. */
if (unlikely(ctx->current_tmp_reg == 8)) {
imm_tmp = (imm64 >> 60) & 0xf; pr_err("eBPF JIT %s[%d]: not enough temporary registers!\n",
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx); current->comm, current->pid);
emit(SW64_BPF_SLL_IMM(dst, 60, dst), ctx); return -1;
}
imm_tmp = (imm64 >> 45) & 0x7fff; return ctx->current_tmp_reg;
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); }
emit(SW64_BPF_SLL_IMM(reg_tmp, 45, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
imm_tmp = (imm64 >> 30) & 0x7fff;
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx);
emit(SW64_BPF_SLL_IMM(reg_tmp, 30, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
imm_tmp = (imm64 >> 15) & 0x7fff;
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx);
emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
imm_tmp = imm64 & 0x7fff; static inline void put_tmp_reg(struct jit_ctx *ctx)
emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx); {
ctx->current_tmp_reg--;
if (ctx->current_tmp_reg == 21)
ctx->current_tmp_reg = 7;
} }
static inline void emit_sw64_ldu32(const int dst, const u32 imm32, struct jit_ctx *ctx) static void emit_sw64_ldu32(const int dst, const u32 imm, struct jit_ctx *ctx)
{ {
u16 imm_tmp; u16 imm_tmp;
int reg_tmp = SW64_BPF_REG_T8; u8 reg_tmp = get_tmp_reg(ctx);
if (!imm) {
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx);
put_tmp_reg(ctx);
return;
}
if (imm <= S16_MAX) {
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx);
put_tmp_reg(ctx);
return;
}
if (imm >= U32_MAX - S16_MAX) {
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
put_tmp_reg(ctx);
return;
}
imm_tmp = (imm32 >> 30) & 3; imm_tmp = (imm >> 30) & 3;
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx); emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx);
emit(SW64_BPF_SLL_IMM(dst, 30, dst), ctx); if (imm_tmp)
emit(SW64_BPF_SLL_IMM(dst, 30, dst), ctx);
imm_tmp = (imm32 >> 15) & 0x7fff; imm_tmp = (imm >> 15) & 0x7fff;
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); if (imm_tmp) {
emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
}
imm_tmp = imm32 & 0x7fff; imm_tmp = imm & 0x7fff;
emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx); if (imm_tmp)
emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx);
put_tmp_reg(ctx);
} }
static inline void emit_sw64_lds32(const int dst, const s32 imm32, struct jit_ctx *ctx) static void emit_sw64_lds32(const int dst, const s32 imm, struct jit_ctx *ctx)
{ {
s16 hi = imm32 >> 16; s16 hi = imm >> 16;
s16 lo = imm32 & 0xffff; s16 lo = imm & 0xffff;
int reg_tmp = SW64_BPF_REG_T8; u8 reg_tmp = get_tmp_reg(ctx);
if (!imm) {
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx);
put_tmp_reg(ctx);
return;
}
if (imm >= S16_MIN && imm <= S16_MAX) {
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx);
put_tmp_reg(ctx);
return;
}
emit(SW64_BPF_LDIH(dst, SW64_BPF_REG_ZR, hi), ctx); emit(SW64_BPF_LDIH(dst, SW64_BPF_REG_ZR, hi), ctx);
if (lo & 0x8000) { // sign bit is 1 if (lo & 0x8000) { // sign bit is 1
...@@ -183,106 +211,299 @@ static inline void emit_sw64_lds32(const int dst, const s32 imm32, struct jit_ct ...@@ -183,106 +211,299 @@ static inline void emit_sw64_lds32(const int dst, const s32 imm32, struct jit_ct
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, 1), ctx); emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, 1), ctx);
emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
emit(SW64_BPF_LDI(dst, dst, lo), ctx); if (lo)
emit(SW64_BPF_LDI(dst, dst, lo), ctx);
} else { // sign bit is 0 } else { // sign bit is 0
emit(SW64_BPF_LDI(dst, dst, lo), ctx); if (lo)
emit(SW64_BPF_LDI(dst, dst, lo), ctx);
}
put_tmp_reg(ctx);
}
static void emit_sw64_ldu64(const int dst, const u64 imm, struct jit_ctx *ctx)
{
u16 imm_tmp;
u8 reg_tmp = get_tmp_reg(ctx);
if (!imm) {
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx);
put_tmp_reg(ctx);
return;
}
if (imm <= U32_MAX) {
put_tmp_reg(ctx);
return emit_sw64_ldu32(dst, (u32)imm, ctx);
}
if (imm >= (U64_MAX - S16_MAX) || imm <= S16_MAX) {
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx);
put_tmp_reg(ctx);
return;
}
imm_tmp = (imm >> 60) & 0xf;
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx);
if (imm_tmp)
emit(SW64_BPF_SLL_IMM(dst, 60, dst), ctx);
imm_tmp = (imm >> 45) & 0x7fff;
if (imm_tmp) {
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx);
emit(SW64_BPF_SLL_IMM(reg_tmp, 45, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
}
imm_tmp = (imm >> 30) & 0x7fff;
if (imm_tmp) {
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx);
emit(SW64_BPF_SLL_IMM(reg_tmp, 30, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
} }
imm_tmp = (imm >> 15) & 0x7fff;
if (imm_tmp) {
emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx);
emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx);
emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx);
}
imm_tmp = imm & 0x7fff;
if (imm_tmp)
emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx);
put_tmp_reg(ctx);
} }
/* dst = ra / rb */ /* Do not change!!! See arch/sw_64/lib/divide.S for more detail */
static void emit_sw64_div(const int ra, const int rb, const int dst, struct jit_ctx *ctx) #define REG(x) "$"str(x)
#define str(x) #x
#define DIVIDEND 24
#define DIVISOR 25
#define RESULT 27
/* Make these functions noinline because we need their address at runtime */
noinline void sw64_bpf_jit_helper_div32(void)
{ {
pr_err("DIV is not supported for now.\n"); register u32 __dividend asm(REG(DIVIDEND));
register u32 __divisor asm(REG(DIVISOR));
u32 res = __dividend / __divisor;
asm volatile(
""
:: "r"(res));
} }
/* dst = ra % rb */ noinline void sw64_bpf_jit_helper_mod32(void)
static void emit_sw64_mod(const int ra, const int rb, const int dst, struct jit_ctx *ctx)
{ {
pr_err("MOD is not supported for now.\n"); register u32 __dividend asm(REG(DIVIDEND));
register u32 __divisor asm(REG(DIVISOR));
u32 res = __dividend % __divisor;
asm volatile(
""
:: "r"(res));
}
noinline void sw64_bpf_jit_helper_div64(void)
{
register s64 __dividend asm(REG(DIVIDEND));
register s64 __divisor asm(REG(DIVISOR));
s64 res = __dividend / __divisor;
asm volatile(
""
:: "r"(res));
}
noinline void sw64_bpf_jit_helper_mod64(void)
{
register s64 __dividend asm(REG(DIVIDEND));
register s64 __divisor asm(REG(DIVISOR));
s64 res = __dividend % __divisor;
asm volatile(
""
:: "r"(res));
}
static void emit_sw64_divmod(const int dst, const int src, struct jit_ctx *ctx, u8 code)
{
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, dst, DIVIDEND), ctx);
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, src, DIVISOR), ctx);
switch (BPF_CLASS(code)) {
case BPF_ALU:
switch (BPF_OP(code)) {
case BPF_DIV:
emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)sw64_bpf_jit_helper_div32, ctx);
break;
case BPF_MOD:
emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)sw64_bpf_jit_helper_mod32, ctx);
break;
}
emit(SW64_BPF_CALL(SW64_BPF_REG_RA, SW64_BPF_REG_PV), ctx);
emit(SW64_BPF_ZAP_IMM(RESULT, 0xf0, dst), ctx);
break;
case BPF_ALU64:
switch (BPF_OP(code)) {
case BPF_DIV:
emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)sw64_bpf_jit_helper_div64, ctx);
break;
case BPF_MOD:
emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)sw64_bpf_jit_helper_mod64, ctx);
break;
}
emit(SW64_BPF_CALL(SW64_BPF_REG_RA, SW64_BPF_REG_PV), ctx);
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, RESULT, dst), ctx);
break;
}
}
#undef REG
#undef str
#undef DIVIDEND
#undef DIVISOR
#undef RESULT
/* STX XADD: lock *(u32 *)(dst + off) += src */
static void emit_sw64_xadd32(const int src, int dst, s16 off, struct jit_ctx *ctx)
{
int atomic_start;
int atomic_end;
u8 tmp1 = get_tmp_reg(ctx);
u8 tmp2 = get_tmp_reg(ctx);
u8 tmp3 = get_tmp_reg(ctx);
if (off < -0x800 || off > 0x7ff) {
emit(SW64_BPF_LDI(tmp1, dst, off), ctx);
dst = tmp1;
off = 0;
}
atomic_start = ctx->idx;
emit(SW64_BPF_LLDW(tmp2, dst, off), ctx);
emit(SW64_BPF_LDI(tmp3, SW64_BPF_REG_ZR, 1), ctx);
emit(SW64_BPF_WR_F(tmp3), ctx);
emit(SW64_BPF_ADDW_REG(tmp2, src, tmp2), ctx);
if (ctx->idx & 1)
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx);
emit(SW64_BPF_LSTW(tmp2, dst, off), ctx);
emit(SW64_BPF_RD_F(tmp3), ctx);
atomic_end = ctx->idx;
emit(SW64_BPF_BEQ(tmp3, atomic_start - atomic_end - 1), ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
}
/* STX XADD: lock *(u64 *)(dst + off) += src */
static void emit_sw64_xadd64(const int src, int dst, s16 off, struct jit_ctx *ctx)
{
int atomic_start;
int atomic_end;
u8 tmp1 = get_tmp_reg(ctx);
u8 tmp2 = get_tmp_reg(ctx);
u8 tmp3 = get_tmp_reg(ctx);
if (off < -0x800 || off > 0x7ff) {
emit(SW64_BPF_LDI(tmp1, dst, off), ctx);
dst = tmp1;
off = 0;
}
atomic_start = ctx->idx;
emit(SW64_BPF_LLDL(tmp2, dst, off), ctx);
emit(SW64_BPF_LDI(tmp3, SW64_BPF_REG_ZR, 1), ctx);
emit(SW64_BPF_WR_F(tmp3), ctx);
emit(SW64_BPF_ADDL_REG(tmp2, src, tmp2), ctx);
if (ctx->idx & 1)
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx);
emit(SW64_BPF_LSTL(tmp2, dst, off), ctx);
emit(SW64_BPF_RD_F(tmp3), ctx);
atomic_end = ctx->idx;
emit(SW64_BPF_BEQ(tmp3, atomic_start - atomic_end - 1), ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
} }
static void emit_sw64_htobe16(const int dst, struct jit_ctx *ctx) static void emit_sw64_htobe16(const int dst, struct jit_ctx *ctx)
{ {
int tmp = SW64_BPF_REG_T8; u8 tmp = get_tmp_reg(ctx);
emit(SW64_BPF_LDI(tmp, dst, 0), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x2, tmp), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp, 0x2, tmp), ctx);
emit(SW64_BPF_ZAPNOT_IMM(dst, 0x1, dst), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x1, dst), ctx);
emit(SW64_BPF_SRL_REG(tmp, 8, tmp), ctx); emit(SW64_BPF_SRL_IMM(tmp, 8, tmp), ctx);
emit(SW64_BPF_SLL_REG(dst, 8, dst), ctx); emit(SW64_BPF_SLL_IMM(dst, 8, dst), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp, dst), ctx); emit(SW64_BPF_BIS_REG(dst, tmp, dst), ctx);
put_tmp_reg(ctx);
} }
static void emit_sw64_htobe32(const int dst, struct jit_ctx *ctx) static void emit_sw64_htobe32(const int dst, struct jit_ctx *ctx)
{ {
int tmp1 = SW64_BPF_REG_T8; u8 tmp1 = get_tmp_reg(ctx);
int tmp2 = SW64_BPF_REG_T9; u8 tmp2 = get_tmp_reg(ctx);
emit(SW64_BPF_LDI(tmp1, dst, 0), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x8, tmp1), ctx);
emit(SW64_BPF_LDI(tmp2, dst, 0), ctx); emit(SW64_BPF_SRL_IMM(tmp1, 24, tmp2), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp1, 0x1, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp2, 0x8, tmp1), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x4, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(dst, 0x6, dst), ctx); emit(SW64_BPF_SRL_IMM(tmp1, 8, tmp1), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 24, tmp1), ctx); emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_SRL_IMM(tmp2, 24, tmp2), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x2, tmp1), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp2, dst), ctx);
emit(SW64_BPF_LDI(tmp1, dst, 0), ctx);
emit(SW64_BPF_LDI(tmp2, dst, 0), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp1, 0x2, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp2, 0x4, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(dst, 0x9, dst), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 8, tmp1), ctx); emit(SW64_BPF_SLL_IMM(tmp1, 8, tmp1), ctx);
emit(SW64_BPF_SRL_IMM(tmp2, 8, tmp2), ctx); emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp2, dst), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x1, dst), ctx);
emit(SW64_BPF_SLL_IMM(dst, 24, dst), ctx);
emit(SW64_BPF_BIS_REG(dst, tmp2, dst), ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
} }
static void emit_sw64_htobe64(const int dst, struct jit_ctx *ctx) static void emit_sw64_htobe64(const int dst, struct jit_ctx *ctx)
{ {
int tmp1 = SW64_BPF_REG_T8; u8 tmp1 = get_tmp_reg(ctx);
int tmp2 = SW64_BPF_REG_T9; u8 tmp2 = get_tmp_reg(ctx);
emit(SW64_BPF_LDI(tmp1, dst, 0), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x80, tmp1), ctx);
emit(SW64_BPF_LDI(tmp2, dst, 0), ctx); emit(SW64_BPF_SRL_IMM(tmp1, 56, tmp2), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp1, 0x1, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp2, 0x80, tmp1), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x40, tmp1), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0x81, dst), ctx); emit(SW64_BPF_SRL_IMM(tmp1, 40, tmp1), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 56, tmp1), ctx); emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_SRL_IMM(tmp2, 56, tmp2), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x20, tmp1), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp2, dst), ctx); emit(SW64_BPF_SRL_IMM(tmp1, 24, tmp1), ctx);
emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_LDI(tmp1, dst, 0), ctx);
emit(SW64_BPF_LDI(tmp2, dst, 0), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x10, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp1, 0x2, tmp1), ctx); emit(SW64_BPF_SRL_IMM(tmp1, 8, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp2, 0x40, tmp1), ctx); emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0x42, dst), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 40, tmp1), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x08, tmp1), ctx);
emit(SW64_BPF_SRL_IMM(tmp2, 40, tmp2), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp2, dst), ctx);
emit(SW64_BPF_LDI(tmp1, dst, 0), ctx);
emit(SW64_BPF_LDI(tmp2, dst, 0), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp1, 0x4, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp2, 0x20, tmp1), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0x24, dst), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 24, tmp1), ctx);
emit(SW64_BPF_SRL_IMM(tmp2, 24, tmp2), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp2, dst), ctx);
emit(SW64_BPF_LDI(tmp1, dst, 0), ctx);
emit(SW64_BPF_LDI(tmp2, dst, 0), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp1, 0x8, tmp1), ctx);
emit(SW64_BPF_ZAPNOT_IMM(tmp2, 0x10, tmp1), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0x18, dst), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 8, tmp1), ctx); emit(SW64_BPF_SLL_IMM(tmp1, 8, tmp1), ctx);
emit(SW64_BPF_SRL_IMM(tmp2, 8, tmp2), ctx); emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp2, dst), ctx); emit(SW64_BPF_ZAPNOT_IMM(dst, 0x04, tmp1), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 24, tmp1), ctx);
emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_ZAPNOT_IMM(dst, 0x02, tmp1), ctx);
emit(SW64_BPF_SLL_IMM(tmp1, 40, tmp1), ctx);
emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx);
emit(SW64_BPF_ZAPNOT_IMM(dst, 0x01, dst), ctx);
emit(SW64_BPF_SLL_IMM(dst, 56, dst), ctx);
emit(SW64_BPF_BIS_REG(dst, tmp2, dst), ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
} }
static void jit_fill_hole(void *area, unsigned int size) static void jit_fill_hole(void *area, unsigned int size)
...@@ -290,107 +511,117 @@ static void jit_fill_hole(void *area, unsigned int size) ...@@ -290,107 +511,117 @@ static void jit_fill_hole(void *area, unsigned int size)
memset(area, SW64_BPF_ILLEGAL_INSN, size); memset(area, SW64_BPF_ILLEGAL_INSN, size);
} }
static int bpf2sw64_offset(int bpf_idx, s32 off, const struct jit_ctx *ctx)
{
int from = ctx->insn_offset[bpf_idx];
int to = ctx->insn_offset[bpf_idx + off];
if (ctx->image == NULL)
return 0;
return to - from;
}
static int offset_to_epilogue(const struct jit_ctx *ctx) static int offset_to_epilogue(const struct jit_ctx *ctx)
{ {
if (ctx->image == NULL)
return 0;
return ctx->epilogue_offset - ctx->idx; return ctx->epilogue_offset - ctx->idx;
} }
/* For tail call to jump into */ /* For tail call, jump to set up function call stack */
#define PROLOGUE_OFFSET 8 #define PROLOGUE_OFFSET 11
static void build_prologue(struct jit_ctx *ctx, bool was_classic) static void build_prologue(struct jit_ctx *ctx, bool was_classic)
{ {
const int r6 = bpf2sw64[BPF_REG_6]; const u8 r6 = bpf2sw64[BPF_REG_6];
const int r7 = bpf2sw64[BPF_REG_7]; const u8 r7 = bpf2sw64[BPF_REG_7];
const int r8 = bpf2sw64[BPF_REG_8]; const u8 r8 = bpf2sw64[BPF_REG_8];
const int r9 = bpf2sw64[BPF_REG_9]; const u8 r9 = bpf2sw64[BPF_REG_9];
const int fp = bpf2sw64[BPF_REG_FP]; const u8 fp = bpf2sw64[BPF_REG_FP];
const int tcc = bpf2sw64[TCALL_CNT]; const u8 tcc = bpf2sw64[TCALL_CNT];
const int tmp1 = bpf2sw64[TMP_REG_1];
/* Save callee-saved registers */ /* Save callee-saved registers */
emit(SW64_BPF_SUBL_REG(SW64_BPF_REG_SP, 56, SW64_BPF_REG_SP), ctx); emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, -64), ctx);
emit(SW64_BPF_STL(r6, SW64_BPF_REG_SP, 0), ctx); emit(SW64_BPF_STL(SW64_BPF_REG_RA, SW64_BPF_REG_SP, 0), ctx);
emit(SW64_BPF_STL(r7, SW64_BPF_REG_SP, 8), ctx); emit(SW64_BPF_STL(fp, SW64_BPF_REG_SP, 8), ctx);
emit(SW64_BPF_STL(r8, SW64_BPF_REG_SP, 16), ctx); emit(SW64_BPF_STL(r6, SW64_BPF_REG_SP, 16), ctx);
emit(SW64_BPF_STL(r9, SW64_BPF_REG_SP, 24), ctx); emit(SW64_BPF_STL(r7, SW64_BPF_REG_SP, 24), ctx);
emit(SW64_BPF_STL(fp, SW64_BPF_REG_SP, 32), ctx); emit(SW64_BPF_STL(r8, SW64_BPF_REG_SP, 32), ctx);
emit(SW64_BPF_STL(tcc, SW64_BPF_REG_SP, 40), ctx); emit(SW64_BPF_STL(r9, SW64_BPF_REG_SP, 40), ctx);
emit(SW64_BPF_STL(SW64_BPF_REG_RA, SW64_BPF_REG_SP, 48), ctx); emit(SW64_BPF_STL(tcc, SW64_BPF_REG_SP, 48), ctx);
emit(SW64_BPF_STL(SW64_BPF_REG_GP, SW64_BPF_REG_SP, 56), ctx);
/* Set up BPF prog stack base register */ /* Set up BPF prog stack base register */
emit(SW64_BPF_LDI(fp, SW64_BPF_REG_SP, 0), ctx); emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_SP, fp), ctx);
if (!was_classic) if (!was_classic)
/* Initialize tail_call_cnt */ /* Initialize tail_call_cnt */
emit(SW64_BPF_LDI(tcc, SW64_BPF_REG_ZR, 0), ctx); emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, tcc), ctx);
/* Set up function call stack */ /* Set up function call stack */
ctx->stack_size = ctx->prog->aux->stack_depth; ctx->stack_size = (ctx->prog->aux->stack_depth + 15) & (~15);
emit_sw64_ldu32(tmp1, ctx->stack_size, ctx); emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, -ctx->stack_size), ctx);
emit(SW64_BPF_SUBL_REG(SW64_BPF_REG_SP, tmp1, SW64_BPF_REG_SP), ctx);
} }
static void build_epilogue(struct jit_ctx *ctx) static void build_epilogue(struct jit_ctx *ctx)
{ {
const int r6 = bpf2sw64[BPF_REG_6]; const u8 r6 = bpf2sw64[BPF_REG_6];
const int r7 = bpf2sw64[BPF_REG_7]; const u8 r7 = bpf2sw64[BPF_REG_7];
const int r8 = bpf2sw64[BPF_REG_8]; const u8 r8 = bpf2sw64[BPF_REG_8];
const int r9 = bpf2sw64[BPF_REG_9]; const u8 r9 = bpf2sw64[BPF_REG_9];
const int fp = bpf2sw64[BPF_REG_FP]; const u8 fp = bpf2sw64[BPF_REG_FP];
const int tcc = bpf2sw64[TCALL_CNT]; const u8 tcc = bpf2sw64[TCALL_CNT];
const int tmp1 = bpf2sw64[TMP_REG_1];
/* Destroy function call stack */ /* Destroy function call stack */
emit_sw64_ldu32(tmp1, ctx->stack_size, ctx); emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, ctx->stack_size), ctx);
emit(SW64_BPF_ADDL_REG(SW64_BPF_REG_SP, tmp1, SW64_BPF_REG_SP), ctx);
/* Restore callee-saved registers */ /* Restore callee-saved registers */
emit(SW64_BPF_LDL(r6, SW64_BPF_REG_SP, 0), ctx); emit(SW64_BPF_LDL(SW64_BPF_REG_RA, SW64_BPF_REG_SP, 0), ctx);
emit(SW64_BPF_LDL(r7, SW64_BPF_REG_SP, 8), ctx); emit(SW64_BPF_LDL(fp, SW64_BPF_REG_SP, 8), ctx);
emit(SW64_BPF_LDL(r8, SW64_BPF_REG_SP, 16), ctx); emit(SW64_BPF_LDL(r6, SW64_BPF_REG_SP, 16), ctx);
emit(SW64_BPF_LDL(r9, SW64_BPF_REG_SP, 24), ctx); emit(SW64_BPF_LDL(r7, SW64_BPF_REG_SP, 24), ctx);
emit(SW64_BPF_LDL(fp, SW64_BPF_REG_SP, 32), ctx); emit(SW64_BPF_LDL(r8, SW64_BPF_REG_SP, 32), ctx);
emit(SW64_BPF_LDL(tcc, SW64_BPF_REG_SP, 40), ctx); emit(SW64_BPF_LDL(r9, SW64_BPF_REG_SP, 40), ctx);
emit(SW64_BPF_LDL(SW64_BPF_REG_RA, SW64_BPF_REG_SP, 48), ctx); emit(SW64_BPF_LDL(tcc, SW64_BPF_REG_SP, 48), ctx);
emit(SW64_BPF_ADDL_REG(SW64_BPF_REG_SP, 56, SW64_BPF_REG_SP), ctx); emit(SW64_BPF_LDL(SW64_BPF_REG_GP, SW64_BPF_REG_SP, 56), ctx);
emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, 64), ctx);
/* Return */ /* Return */
emit(SW64_BPF_RET(SW64_BPF_REG_RA), ctx); emit(SW64_BPF_RET(SW64_BPF_REG_RA), ctx);
} }
static int out_offset = -1; /* initialized on the first pass of build_body() */
static int emit_bpf_tail_call(struct jit_ctx *ctx) static int emit_bpf_tail_call(struct jit_ctx *ctx)
{ {
/* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */
const u8 r2 = bpf2sw64[BPF_REG_2]; /* struct bpf_array *array */ const u8 r2 = bpf2sw64[BPF_REG_2]; /* struct bpf_array *array */
const u8 r3 = bpf2sw64[BPF_REG_3]; /* u64 index */ const u8 r3 = bpf2sw64[BPF_REG_3]; /* u64 index */
const u8 tmp = bpf2sw64[TMP_REG_1]; const u8 tmp = get_tmp_reg(ctx);
const u8 prg = bpf2sw64[TMP_REG_2]; const u8 prg = get_tmp_reg(ctx);
const u8 tcc = bpf2sw64[TCALL_CNT]; const u8 tcc = bpf2sw64[TCALL_CNT];
const int idx0 = ctx->idx;
#define cur_offset (ctx->idx - idx0)
#define jmp_offset (out_offset - (cur_offset))
u64 offset; u64 offset;
static int out_idx;
#define out_offset (ctx->image ? (out_idx - ctx->idx - 1) : 0)
/* if (index >= array->map.max_entries) /* if (index >= array->map.max_entries)
* goto out; * goto out;
*/ */
offset = offsetof(struct bpf_array, map.max_entries); offset = offsetof(struct bpf_array, map.max_entries);
emit_sw64_ldu64(tmp, offset, ctx); /* tmp = offset */ emit_sw64_ldu64(tmp, offset, ctx);
emit(SW64_BPF_ADDL_REG(r2, tmp, tmp), ctx); /* tmp = r2 + tmp = &map.max_entries */ emit(SW64_BPF_ADDL_REG(r2, tmp, tmp), ctx); /* tmp = r2 + tmp = &map.max_entries */
emit(SW64_BPF_LDW(tmp, tmp, 0), ctx); /* tmp = *tmp = map.max_entries */ emit(SW64_BPF_LDW(tmp, tmp, 0), ctx); /* tmp = *tmp = map.max_entries */
emit(SW64_BPF_ZAPNOT_IMM(tmp, 0xf, tmp), ctx); /* map.max_entries is u32 */ emit(SW64_BPF_ZAP_IMM(tmp, 0xf0, tmp), ctx); /* map.max_entries is u32 */
emit(SW64_BPF_SUBL_REG(r3, tmp, tmp), ctx); /* tmp = r3 - tmp = index - map.max_entries */ emit(SW64_BPF_CMPULE_REG(tmp, r3, tmp), ctx);
emit(SW64_BPF_BGE(tmp, jmp_offset), ctx); emit(SW64_BPF_BNE(tmp, out_offset), ctx);
/* if (tail_call_cnt > MAX_TAIL_CALL_CNT) /* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
* goto out; * goto out;
* tail_call_cnt++; * tail_call_cnt++;
*/ */
emit(SW64_BPF_LDI(tmp, SW64_BPF_REG_ZR, MAX_TAIL_CALL_CNT), ctx); emit_sw64_ldu64(tmp, MAX_TAIL_CALL_CNT, ctx);
emit(SW64_BPF_SUBL_REG(tcc, tmp, tmp), ctx); emit(SW64_BPF_CMPULE_REG(tcc, tmp, tmp), ctx);
emit(SW64_BPF_BGT(tmp, jmp_offset), ctx); emit(SW64_BPF_BEQ(tmp, out_offset), ctx);
emit(SW64_BPF_ADDL_IMM(tcc, 1, tcc), ctx); emit(SW64_BPF_ADDL_IMM(tcc, 1, tcc), ctx);
/* prog = array->ptrs[index]; /* prog = array->ptrs[index];
...@@ -398,34 +629,33 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) ...@@ -398,34 +629,33 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
* goto out; * goto out;
*/ */
offset = offsetof(struct bpf_array, ptrs); offset = offsetof(struct bpf_array, ptrs);
emit_sw64_ldu64(tmp, offset, ctx); /* tmp = offset of ptrs */ emit_sw64_ldu64(tmp, offset, ctx);
emit(SW64_BPF_ADDL_REG(r2, tmp, tmp), ctx); /* tmp = r2 + tmp = &ptrs */ emit(SW64_BPF_ADDL_REG(r2, tmp, tmp), ctx); /* tmp = r2 + tmp = &ptrs[0] */
emit(SW64_BPF_SLL_IMM(r3, 3, prg), ctx); /* prg = r3 * 8, ptrs is 8 bit aligned */ emit(SW64_BPF_SLL_IMM(r3, 3, prg), ctx); /* prg = r3 * 8, each entry is a pointer */
emit(SW64_BPF_ADDL_REG(tmp, prg, prg), ctx); /* prg = tmp + prg = &prog */ emit(SW64_BPF_ADDL_REG(tmp, prg, prg), ctx); /* prg = tmp + prg = &ptrs[index] */
emit(SW64_BPF_LDL(prg, prg, 0), ctx); /* prg = *prg = prog */ emit(SW64_BPF_LDL(prg, prg, 0), ctx); /* prg = *prg = ptrs[index] = prog */
emit(SW64_BPF_BEQ(prg, jmp_offset), ctx); emit(SW64_BPF_BEQ(prg, out_offset), ctx);
/* goto *(prog->bpf_func + prologue_offset); */ /* goto *(prog->bpf_func + prologue_offset); */
offset = offsetof(struct bpf_prog, bpf_func); offset = offsetof(struct bpf_prog, bpf_func);
emit_sw64_ldu64(tmp, offset, ctx); /* tmp = offset */ emit_sw64_ldu64(tmp, offset, ctx);
emit(SW64_BPF_ADDL_REG(prg, tmp, tmp), ctx); /* tmp = prg + tmp = &bpf_func */ emit(SW64_BPF_ADDL_REG(prg, tmp, tmp), ctx); /* tmp = prg + tmp = &bpf_func */
emit(SW64_BPF_LDW(tmp, tmp, 0), ctx); /* tmp = *tmp = bpf_func */ emit(SW64_BPF_LDL(tmp, tmp, 0), ctx); /* tmp = *tmp = bpf_func */
emit(SW64_BPF_ZAPNOT_IMM(tmp, 0xf, tmp), ctx); /* bpf_func is unsigned int */ emit(SW64_BPF_BEQ(tmp, out_offset), ctx);
emit(SW64_BPF_ADDL_REG(tmp, sizeof(u32) * PROLOGUE_OFFSET, tmp), ctx); emit(SW64_BPF_ADDL_REG(tmp, sizeof(u32) * PROLOGUE_OFFSET, tmp), ctx);
emit(SW64_BPF_ADDL_REG(SW64_BPF_REG_SP, ctx->stack_size, SW64_BPF_REG_SP), ctx); emit(SW64_BPF_ADDL_REG(SW64_BPF_REG_SP, ctx->stack_size, SW64_BPF_REG_SP), ctx);
emit(SW64_BPF_BR(tmp, 0), ctx); emit(SW64_BPF_JMP(SW64_BPF_REG_ZR, tmp), ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
/* out */ /* out */
if (out_offset == -1) if (ctx->image == NULL)
out_offset = cur_offset; out_idx = ctx->idx;
if (cur_offset != out_offset) { if (ctx->image != NULL && out_offset <= 0)
pr_err("tail_call out_offset = %d, expected %d!\n",
cur_offset, out_offset);
return -1; return -1;
} #undef out_offset
return 0; return 0;
#undef cur_offset
#undef jmp_offset
} }
/* JITs an eBPF instruction. /* JITs an eBPF instruction.
...@@ -434,61 +664,79 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) ...@@ -434,61 +664,79 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
* >0 - successfully JITed a 16-byte eBPF instruction. * >0 - successfully JITed a 16-byte eBPF instruction.
* <0 - failed to JIT. * <0 - failed to JIT.
*/ */
static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
{ {
const u8 code = insn->code; const u8 code = insn->code;
const u8 dst = bpf2sw64[insn->dst_reg]; const u8 dst = bpf2sw64[insn->dst_reg];
const u8 src = bpf2sw64[insn->src_reg]; const u8 src = bpf2sw64[insn->src_reg];
const u8 tmp1 = bpf2sw64[TMP_REG_1]; const u8 tmp1 __maybe_unused = get_tmp_reg(ctx);
const u8 tmp2 = bpf2sw64[TMP_REG_2]; const u8 tmp2 __maybe_unused = get_tmp_reg(ctx);
const s16 off = insn->off; const s16 off = insn->off;
const s32 imm = insn->imm; const s32 imm = insn->imm;
int jmp_offset; const int bpf_idx = insn - ctx->prog->insnsi;
s32 jmp_offset;
u64 func; u64 func;
struct bpf_insn insn1; struct bpf_insn insn1;
u64 imm64; u64 imm64;
switch (code) { switch (code) {
case BPF_ALU | BPF_MOV | BPF_X: case BPF_ALU | BPF_MOV | BPF_X:
emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, src, dst), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_MOV | BPF_X: case BPF_ALU64 | BPF_MOV | BPF_X:
emit(SW64_BPF_LDI(dst, src, 0), ctx); emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, src, dst), ctx);
break; break;
case BPF_ALU | BPF_ADD | BPF_X: case BPF_ALU | BPF_ADD | BPF_X:
emit(SW64_BPF_ADDW_REG(dst, src, dst), ctx); emit(SW64_BPF_ADDW_REG(dst, src, dst), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break; break;
case BPF_ALU64 | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_X:
emit(SW64_BPF_ADDL_REG(dst, src, dst), ctx); emit(SW64_BPF_ADDL_REG(dst, src, dst), ctx);
break; break;
case BPF_ALU | BPF_SUB | BPF_X: case BPF_ALU | BPF_SUB | BPF_X:
emit(SW64_BPF_SUBW_REG(dst, src, dst), ctx); emit(SW64_BPF_SUBW_REG(dst, src, dst), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break; break;
case BPF_ALU64 | BPF_SUB | BPF_X: case BPF_ALU64 | BPF_SUB | BPF_X:
emit(SW64_BPF_SUBL_REG(dst, src, dst), ctx); emit(SW64_BPF_SUBL_REG(dst, src, dst), ctx);
break; break;
case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU | BPF_MUL | BPF_X:
emit(SW64_BPF_MULW_REG(dst, src, dst), ctx); emit(SW64_BPF_MULW_REG(dst, src, dst), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break; break;
case BPF_ALU64 | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X:
emit(SW64_BPF_MULL_REG(dst, src, dst), ctx); emit(SW64_BPF_MULL_REG(dst, src, dst), ctx);
break; break;
case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_DIV | BPF_X:
emit_sw64_divmod(dst, src, ctx, code);
break;
case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X:
emit_sw64_div(dst, src, dst, ctx); emit_sw64_divmod(dst, src, ctx, code);
return -EINVAL; break;
case BPF_ALU | BPF_MOD | BPF_X: case BPF_ALU | BPF_MOD | BPF_X:
emit_sw64_divmod(dst, src, ctx, code);
break;
case BPF_ALU64 | BPF_MOD | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X:
emit_sw64_mod(dst, src, dst, ctx); emit_sw64_divmod(dst, src, ctx, code);
return -EINVAL; break;
case BPF_ALU | BPF_LSH | BPF_X: case BPF_ALU | BPF_LSH | BPF_X:
emit(SW64_BPF_SLL_REG(dst, src, dst), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_LSH | BPF_X:
emit(SW64_BPF_SLL_REG(dst, src, dst), ctx); emit(SW64_BPF_SLL_REG(dst, src, dst), ctx);
break; break;
case BPF_ALU | BPF_RSH | BPF_X: case BPF_ALU | BPF_RSH | BPF_X:
emit(SW64_BPF_ZAPNOT_IMM(dst, 0xf, dst), ctx); emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
case BPF_ALU64 | BPF_RSH | BPF_X: case BPF_ALU64 | BPF_RSH | BPF_X:
emit(SW64_BPF_SRL_REG(dst, src, dst), ctx); emit(SW64_BPF_SRL_REG(dst, src, dst), ctx);
break; break;
case BPF_ALU | BPF_ARSH | BPF_X: case BPF_ALU | BPF_ARSH | BPF_X:
emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, dst), ctx);
emit(SW64_BPF_SRA_REG(dst, src, dst), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_ARSH | BPF_X: case BPF_ALU64 | BPF_ARSH | BPF_X:
emit(SW64_BPF_SRA_REG(dst, src, dst), ctx); emit(SW64_BPF_SRA_REG(dst, src, dst), ctx);
break; break;
...@@ -498,16 +746,18 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -498,16 +746,18 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
break; break;
case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU | BPF_OR | BPF_X:
case BPF_ALU64 | BPF_OR | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X:
emit(SW64_BPF_OR_REG(dst, src, dst), ctx); emit(SW64_BPF_BIS_REG(dst, src, dst), ctx);
break; break;
case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU | BPF_XOR | BPF_X:
case BPF_ALU64 | BPF_XOR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X:
emit(SW64_BPF_XOR_REG(dst, src, dst), ctx); emit(SW64_BPF_XOR_REG(dst, src, dst), ctx);
break; break;
case BPF_ALU | BPF_NEG: case BPF_ALU | BPF_NEG:
emit(SW64_BPF_SUBW_REG(SW64_BPF_REG_ZR, dst, dst), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_NEG: case BPF_ALU64 | BPF_NEG:
emit(SW64_BPF_SEXTB_IMM(0xff, tmp1), ctx); emit(SW64_BPF_SUBL_REG(SW64_BPF_REG_ZR, dst, dst), ctx);
emit(SW64_BPF_XOR_IMM(dst, tmp1, dst), ctx);
break; break;
case BPF_ALU | BPF_END | BPF_TO_LE: case BPF_ALU | BPF_END | BPF_TO_LE:
switch (imm) { switch (imm) {
...@@ -519,7 +769,12 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -519,7 +769,12 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
break; break;
case 64: case 64:
break; break;
default:
pr_err("eBPF JIT %s[%d]: BPF_TO_LE unknown size\n",
current->comm, current->pid);
return -EINVAL;
} }
break;
case BPF_ALU | BPF_END | BPF_TO_BE: case BPF_ALU | BPF_END | BPF_TO_BE:
switch (imm) { switch (imm) {
case 16: case 16:
...@@ -531,71 +786,203 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -531,71 +786,203 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
case 64: case 64:
emit_sw64_htobe64(dst, ctx); emit_sw64_htobe64(dst, ctx);
break; break;
default:
pr_err("eBPF JIT %s[%d]: BPF_TO_BE unknown size\n",
current->comm, current->pid);
return -EINVAL;
} }
break;
case BPF_ALU | BPF_MOV | BPF_K: case BPF_ALU | BPF_MOV | BPF_K:
if (imm >= S16_MIN && imm <= S16_MAX)
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx);
else
emit_sw64_ldu32(dst, imm, ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_MOV | BPF_K: case BPF_ALU64 | BPF_MOV | BPF_K:
emit_sw64_lds32(dst, imm, ctx); if (imm >= S16_MIN && imm <= S16_MAX)
emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx);
else
emit_sw64_lds32(dst, imm, ctx);
break; break;
case BPF_ALU | BPF_ADD | BPF_K: case BPF_ALU | BPF_ADD | BPF_K:
if (imm >= S16_MIN && imm <= S16_MAX) {
emit(SW64_BPF_LDI(dst, dst, imm), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_ADDW_REG(dst, tmp1, dst), ctx);
}
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_ADD | BPF_K: case BPF_ALU64 | BPF_ADD | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= S16_MIN && imm <= S16_MAX) {
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_LDI(dst, dst, imm), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_SUB | BPF_K: case BPF_ALU | BPF_SUB | BPF_K:
if (imm >= -S16_MAX && imm <= -S16_MIN) {
emit(SW64_BPF_LDI(dst, dst, -imm), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_SUBL_REG(dst, tmp1, dst), ctx);
}
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_SUB | BPF_K: case BPF_ALU64 | BPF_SUB | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= -S16_MAX && imm <= -S16_MIN) {
emit(SW64_BPF_SUBL_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_LDI(dst, dst, -imm), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_SUBL_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_MUL | BPF_K:
if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_MULL_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_MULL_REG(dst, tmp1, dst), ctx);
}
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_MULL_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_MULL_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_MULL_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_DIV | BPF_K:
emit_sw64_ldu32(tmp1, imm, ctx);
emit_sw64_divmod(dst, tmp1, ctx, code);
break;
case BPF_ALU64 | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); emit_sw64_lds32(tmp1, imm, ctx);
emit_sw64_div(dst, src, tmp1, ctx); emit_sw64_divmod(dst, tmp1, ctx, code);
return -EINVAL; break;
case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU | BPF_MOD | BPF_K:
emit_sw64_ldu32(tmp1, imm, ctx);
emit_sw64_divmod(dst, tmp1, ctx, code);
break;
case BPF_ALU64 | BPF_MOD | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); emit_sw64_lds32(tmp1, imm, ctx);
emit_sw64_mod(dst, src, tmp1, ctx); emit_sw64_divmod(dst, tmp1, ctx, code);
return -EINVAL; break;
case BPF_ALU | BPF_LSH | BPF_K: case BPF_ALU | BPF_LSH | BPF_K:
if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_SLL_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_SLL_REG(dst, tmp1, dst), ctx);
}
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_LSH | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_SLL_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_SLL_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_SLL_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_RSH | BPF_K: case BPF_ALU | BPF_RSH | BPF_K:
emit(SW64_BPF_ZAPNOT_IMM(dst, 0xf, dst), ctx); emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_SRL_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_SRL_REG(dst, tmp1, dst), ctx);
}
break;
case BPF_ALU64 | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_SRL_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_SRL_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_SRL_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_ARSH | BPF_K: case BPF_ALU | BPF_ARSH | BPF_K:
emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, dst), ctx);
if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_SRA_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_SRA_REG(dst, tmp1, dst), ctx);
}
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break;
case BPF_ALU64 | BPF_ARSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_SRA_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_SRA_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_SRA_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU | BPF_AND | BPF_K:
if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_AND_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_AND_REG(dst, tmp1, dst), ctx);
}
break;
case BPF_ALU64 | BPF_AND | BPF_K: case BPF_ALU64 | BPF_AND | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_AND_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_AND_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_AND_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_OR | BPF_K: case BPF_ALU | BPF_OR | BPF_K:
if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_BIS_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_BIS_REG(dst, tmp1, dst), ctx);
}
break;
case BPF_ALU64 | BPF_OR | BPF_K: case BPF_ALU64 | BPF_OR | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_OR_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_BIS_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_BIS_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU | BPF_XOR | BPF_K:
if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_XOR_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_ldu32(tmp1, imm, ctx);
emit(SW64_BPF_XOR_REG(dst, tmp1, dst), ctx);
}
break;
case BPF_ALU64 | BPF_XOR | BPF_K: case BPF_ALU64 | BPF_XOR | BPF_K:
emit_sw64_lds32(tmp1, imm, ctx); if (imm >= 0 && imm <= U8_MAX) {
emit(SW64_BPF_XOR_REG(dst, tmp1, dst), ctx); emit(SW64_BPF_XOR_IMM(dst, imm, dst), ctx);
} else {
emit_sw64_lds32(tmp1, imm, ctx);
emit(SW64_BPF_XOR_REG(dst, tmp1, dst), ctx);
}
break; break;
case BPF_JMP | BPF_JA: case BPF_JMP | BPF_JA:
emit(SW64_BPF_BR(SW64_BPF_REG_RA, off), ctx); jmp_offset = bpf2sw64_offset(bpf_idx, off, ctx);
if (jmp_offset >= -0x100000 && jmp_offset <= 0xfffff) {
emit(SW64_BPF_BR(SW64_BPF_REG_ZR, jmp_offset), ctx);
} else {
pr_err("eBPF JIT %s[%d]: BPF_JMP out of range, %d instructions\n",
current->comm, current->pid, jmp_offset);
return -EINVAL;
}
break; break;
case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP | BPF_JEQ | BPF_X:
...@@ -645,7 +1032,14 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -645,7 +1032,14 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit(SW64_BPF_AND_REG(dst, src, tmp1), ctx); emit(SW64_BPF_AND_REG(dst, src, tmp1), ctx);
break; break;
} }
emit(SW64_BPF_BLBS(tmp1, off), ctx); jmp_offset = bpf2sw64_offset(bpf_idx, off, ctx);
if (jmp_offset >= -0x100000 && jmp_offset <= 0xfffff) {
emit(SW64_BPF_BNE(tmp1, jmp_offset), ctx);
} else {
pr_err("eBPF JIT %s[%d]: BPF_JMP out of range, %d instructions\n",
current->comm, current->pid, jmp_offset);
return -EINVAL;
}
break; break;
case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP | BPF_JEQ | BPF_K:
...@@ -662,47 +1056,54 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -662,47 +1056,54 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit_sw64_lds32(tmp1, imm, ctx); emit_sw64_lds32(tmp1, imm, ctx);
switch (BPF_OP(code)) { switch (BPF_OP(code)) {
case BPF_JEQ: case BPF_JEQ:
emit(SW64_BPF_CMPEQ_REG(dst, tmp1, tmp1), ctx); emit(SW64_BPF_CMPEQ_REG(dst, tmp1, tmp2), ctx);
break; break;
case BPF_JGT: case BPF_JGT:
emit(SW64_BPF_CMPULT_REG(tmp1, dst, tmp1), ctx); emit(SW64_BPF_CMPULT_REG(tmp1, dst, tmp2), ctx);
break; break;
case BPF_JLT: case BPF_JLT:
emit(SW64_BPF_CMPULT_REG(dst, tmp1, tmp1), ctx); emit(SW64_BPF_CMPULT_REG(dst, tmp1, tmp2), ctx);
break; break;
case BPF_JGE: case BPF_JGE:
emit(SW64_BPF_CMPULE_REG(tmp1, dst, tmp1), ctx); emit(SW64_BPF_CMPULE_REG(tmp1, dst, tmp2), ctx);
break; break;
case BPF_JLE: case BPF_JLE:
emit(SW64_BPF_CMPULE_REG(dst, tmp1, tmp1), ctx); emit(SW64_BPF_CMPULE_REG(dst, tmp1, tmp2), ctx);
break; break;
case BPF_JNE: case BPF_JNE:
emit(SW64_BPF_CMPEQ_REG(dst, tmp1, tmp1), ctx); emit(SW64_BPF_CMPEQ_REG(dst, tmp1, tmp2), ctx);
emit(SW64_BPF_XOR_IMM(tmp1, 1, tmp1), ctx); emit(SW64_BPF_XOR_IMM(tmp2, 1, tmp2), ctx);
break; break;
case BPF_JSGT: case BPF_JSGT:
emit(SW64_BPF_CMPLT_REG(tmp1, dst, tmp1), ctx); emit(SW64_BPF_CMPLT_REG(tmp1, dst, tmp2), ctx);
break; break;
case BPF_JSLT: case BPF_JSLT:
emit(SW64_BPF_CMPLT_REG(dst, tmp1, tmp1), ctx); emit(SW64_BPF_CMPLT_REG(dst, tmp1, tmp2), ctx);
break; break;
case BPF_JSGE: case BPF_JSGE:
emit(SW64_BPF_CMPLE_REG(tmp1, dst, tmp1), ctx); emit(SW64_BPF_CMPLE_REG(tmp1, dst, tmp2), ctx);
break; break;
case BPF_JSLE: case BPF_JSLE:
emit(SW64_BPF_CMPLE_REG(dst, tmp1, tmp1), ctx); emit(SW64_BPF_CMPLE_REG(dst, tmp1, tmp2), ctx);
break; break;
case BPF_JSET: case BPF_JSET:
emit(SW64_BPF_AND_REG(dst, tmp1, tmp1), ctx); emit(SW64_BPF_AND_REG(dst, tmp1, tmp2), ctx);
break; break;
} }
emit(SW64_BPF_BLBS(tmp1, off), ctx); jmp_offset = bpf2sw64_offset(bpf_idx, off, ctx);
if (jmp_offset >= -0x100000 && jmp_offset <= 0xfffff) {
emit(SW64_BPF_BNE(tmp2, jmp_offset), ctx);
} else {
pr_err("eBPF JIT %s[%d]: BPF_JMP out of range, %d instructions\n",
current->comm, current->pid, jmp_offset);
return -EINVAL;
}
break; break;
case BPF_JMP | BPF_CALL: case BPF_JMP | BPF_CALL:
func = (u64)__bpf_call_base + imm; func = (u64)__bpf_call_base + imm;
emit_sw64_ldu64(tmp1, func, ctx); emit_sw64_ldu64(SW64_BPF_REG_PV, func, ctx);
emit(SW64_BPF_CALL(SW64_BPF_REG_RA, tmp1), ctx); emit(SW64_BPF_CALL(SW64_BPF_REG_RA, SW64_BPF_REG_PV), ctx);
break; break;
case BPF_JMP | BPF_TAIL_CALL: case BPF_JMP | BPF_TAIL_CALL:
...@@ -711,38 +1112,45 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -711,38 +1112,45 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
break; break;
case BPF_JMP | BPF_EXIT: case BPF_JMP | BPF_EXIT:
if (insn - ctx->prog->insnsi + 1 == ctx->prog->len) // if this is the last instruction, fallthrough to epilogue
if (bpf_idx == ctx->prog->len - 1)
break; break;
jmp_offset = (offset_to_epilogue(ctx) - 1) * 4; jmp_offset = offset_to_epilogue(ctx) - 1;
// emit(SW64_BPF_BR(SW64_BPF_REG_ZR, jmp_offset), ctx); // epilogue is always at the end, must jump forward
// break; if (jmp_offset >= -1 && jmp_offset <= 0xfffff) {
emit_sw64_lds32(tmp1, jmp_offset, ctx); if (ctx->image && !jmp_offset)
emit(SW64_BPF_BR(tmp2, 0), ctx); // if this is the last instruction, fallthrough to epilogue
emit(SW64_BPF_ADDL_REG(tmp1, tmp2, tmp1), ctx); emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx);
emit(SW64_BPF_JMP(SW64_BPF_REG_ZR, tmp1), ctx); else
emit(SW64_BPF_BR(SW64_BPF_REG_ZR, jmp_offset), ctx);
} else {
pr_err("eBPF JIT %s[%d]: BPF_EXIT out of range, %d instructions\n",
current->comm, current->pid, jmp_offset);
return -EINVAL;
}
break; break;
case BPF_LD | BPF_IMM | BPF_DW: case BPF_LD | BPF_IMM | BPF_DW:
insn1 = insn[1]; insn1 = insn[1];
imm64 = (u64)insn1.imm << 32 | (u32)imm; imm64 = ((u64)insn1.imm << 32) | (u32)imm;
emit_sw64_ldu64(dst, imm64, ctx); emit_sw64_ldu64(dst, imm64, ctx);
put_tmp_reg(ctx);
put_tmp_reg(ctx);
return 1; return 1;
/* LDX: dst = *(size *)(src + off) */ /* LDX: dst = *(size *)(src + off) */
case BPF_LDX | BPF_MEM | BPF_W: case BPF_LDX | BPF_MEM | BPF_W:
emit(SW64_BPF_LDW(dst, src, off), ctx); emit(SW64_BPF_LDW(dst, src, off), ctx);
emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx);
break; break;
case BPF_LDX | BPF_MEM | BPF_H: case BPF_LDX | BPF_MEM | BPF_H:
emit(SW64_BPF_LDHU(dst, src, off), ctx); emit(SW64_BPF_LDHU(dst, src, off), ctx);
emit(SW64_BPF_SEXTH_REG(dst, dst), ctx);
break; break;
case BPF_LDX | BPF_MEM | BPF_B: case BPF_LDX | BPF_MEM | BPF_B:
emit(SW64_BPF_LDBU(dst, src, off), ctx); emit(SW64_BPF_LDBU(dst, src, off), ctx);
emit(SW64_BPF_SEXTB_REG(dst, dst), ctx);
break; break;
case BPF_LDX | BPF_MEM | BPF_DW: case BPF_LDX | BPF_MEM | BPF_DW:
emit(SW64_BPF_LDW(dst, src, off), ctx); emit(SW64_BPF_LDL(dst, src, off), ctx);
break; break;
/* ST: *(size *)(dst + off) = imm */ /* ST: *(size *)(dst + off) = imm */
...@@ -773,33 +1181,32 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ...@@ -773,33 +1181,32 @@ static inline int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit(SW64_BPF_STW(src, dst, off), ctx); emit(SW64_BPF_STW(src, dst, off), ctx);
break; break;
case BPF_STX | BPF_MEM | BPF_H: case BPF_STX | BPF_MEM | BPF_H:
emit(SW64_BPF_STW(src, dst, off), ctx); emit(SW64_BPF_STH(src, dst, off), ctx);
break; break;
case BPF_STX | BPF_MEM | BPF_B: case BPF_STX | BPF_MEM | BPF_B:
emit(SW64_BPF_STW(src, dst, off), ctx); emit(SW64_BPF_STB(src, dst, off), ctx);
break; break;
case BPF_STX | BPF_MEM | BPF_DW: case BPF_STX | BPF_MEM | BPF_DW:
emit(SW64_BPF_STW(src, dst, off), ctx); emit(SW64_BPF_STL(src, dst, off), ctx);
break; break;
/* STX XADD: lock *(u32 *)(dst + off) += src */ /* STX XADD: lock *(u32 *)(dst + off) += src */
case BPF_STX | BPF_XADD | BPF_W: case BPF_STX | BPF_XADD | BPF_W:
emit(SW64_BPF_LDW(tmp1, dst, off), ctx); emit_sw64_xadd32(src, dst, off, ctx);
emit(SW64_BPF_ADDW_REG(tmp1, src, tmp1), ctx);
emit(SW64_BPF_STW(tmp1, dst, off), ctx);
break; break;
/* STX XADD: lock *(u64 *)(dst + off) += src */ /* STX XADD: lock *(u64 *)(dst + off) += src */
case BPF_STX | BPF_XADD | BPF_DW: case BPF_STX | BPF_XADD | BPF_DW:
emit(SW64_BPF_LDL(tmp1, dst, off), ctx); emit_sw64_xadd64(src, dst, off, ctx);
emit(SW64_BPF_ADDL_REG(tmp1, src, tmp1), ctx);
emit(SW64_BPF_STL(tmp1, dst, off), ctx);
break; break;
default: default:
pr_err("unknown opcode %02x\n", code); pr_err("eBPF JIT %s[%d]: unknown opcode 0x%02x\n",
current->comm, current->pid, code);
return -EINVAL; return -EINVAL;
} }
put_tmp_reg(ctx);
put_tmp_reg(ctx);
return 0; return 0;
} }
...@@ -813,16 +1220,16 @@ static int build_body(struct jit_ctx *ctx) ...@@ -813,16 +1220,16 @@ static int build_body(struct jit_ctx *ctx)
int ret; int ret;
ret = build_insn(insn, ctx); ret = build_insn(insn, ctx);
if (ret > 0) { if (ret < 0)
return ret;
if (ctx->image == NULL)
ctx->insn_offset[i] = ctx->idx;
while (ret > 0) {
i++; i++;
if (ctx->image == NULL) if (ctx->image == NULL)
ctx->insn_offset[i] = ctx->idx; ctx->insn_offset[i] = ctx->idx;
continue; ret--;
} }
if (ctx->image == NULL)
ctx->insn_offset[i] = ctx->idx;
if (ret)
return ret;
} }
return 0; return 0;
...@@ -911,7 +1318,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ...@@ -911,7 +1318,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
build_epilogue(&ctx); build_epilogue(&ctx);
/* Now we know the actual image size. */ /* Now we know the actual image size. */
image_size = sizeof(u32) * ctx.idx; /* And we need extra 8 bytes for lock instructions alignment */
image_size = sizeof(u32) * ctx.idx + 8;
header = bpf_jit_binary_alloc(image_size, &image_ptr, header = bpf_jit_binary_alloc(image_size, &image_ptr,
sizeof(u32), jit_fill_hole); sizeof(u32), jit_fill_hole);
if (header == NULL) { if (header == NULL) {
...@@ -921,7 +1329,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ...@@ -921,7 +1329,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
/* 2. Now, the actual pass. */ /* 2. Now, the actual pass. */
ctx.image = (u32 *)image_ptr; /* lock instructions need 8-byte alignment */
ctx.image = (u32 *)(((unsigned long)image_ptr + 7) & (~7));
skip_init_ctx: skip_init_ctx:
ctx.idx = 0; ctx.idx = 0;
...@@ -958,6 +1367,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ...@@ -958,6 +1367,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
prog->bpf_func = (void *)ctx.image; prog->bpf_func = (void *)ctx.image;
prog->jited = 1; prog->jited = 1;
prog->jited_len = image_size; prog->jited_len = image_size;
if (ctx.current_tmp_reg) {
pr_err("eBPF JIT %s[%d]: unreleased temporary regsters %d\n",
current->comm, current->pid, ctx.current_tmp_reg);
}
if (!prog->is_func || extra_pass) { if (!prog->is_func || extra_pass) {
out_off: out_off:
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或