diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 1d9be26b4edd79b2ad0f4661411790d9e71d45dd..a11d7270583d0f5861e944e26efa12d2b09ab92c 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -282,7 +282,7 @@ $(obj)/hbm_edt_kern.o: $(src)/hbm.h $(src)/hbm_kern.h $(obj)/%.o: $(src)/%.c @echo " CLANG-bpf " $@ $(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(EXTRA_CFLAGS) -I$(obj) \ - -I$(srctree)/tools/testing/selftests/bpf/ \ + -I$(srctree)/tools/testing/selftests/bpf/ -I$(srctree)/tools/lib/bpf/ \ -D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \ -D__TARGET_ARCH_$(SRCARCH) -Wno-compare-distinct-pointer-types \ -Wno-gnu-variable-sized-type-not-at-end \ diff --git a/samples/bpf/hbm_kern.h b/samples/bpf/hbm_kern.h index aa207a2eebbde4b1ea184e475753df155689ea60..4edaf47876ca77d75ca25886749cdb2e8d89932c 100644 --- a/samples/bpf/hbm_kern.h +++ b/samples/bpf/hbm_kern.h @@ -59,21 +59,18 @@ #define BYTES_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20) #define BYTES_TO_NS(bytes, rate) div64_u64(((u64)(bytes)) << 20, (u64)(rate)) -struct bpf_map_def SEC("maps") queue_state = { - .type = BPF_MAP_TYPE_CGROUP_STORAGE, - .key_size = sizeof(struct bpf_cgroup_storage_key), - .value_size = sizeof(struct hbm_vqueue), -}; -BPF_ANNOTATE_KV_PAIR(queue_state, struct bpf_cgroup_storage_key, - struct hbm_vqueue); - -struct bpf_map_def SEC("maps") queue_stats = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(u32), - .value_size = sizeof(struct hbm_queue_stats), - .max_entries = 1, -}; -BPF_ANNOTATE_KV_PAIR(queue_stats, int, struct hbm_queue_stats); +struct { + __uint(type, BPF_MAP_TYPE_CGROUP_STORAGE); + __type(key, struct bpf_cgroup_storage_key); + __type(value, struct hbm_vqueue); +} queue_state SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct hvm_queue_stats); +} queue_stats SEC(".maps"); struct hbm_pkt_info { int cwnd; diff --git a/samples/bpf/map_perf_test_kern.c b/samples/bpf/map_perf_test_kern.c index 2b2ffb97018bd93d91747d08f8496cabab14c771..5c11aefbc489858a62c4bd4ea26ff248caf9304e 100644 --- a/samples/bpf/map_perf_test_kern.c +++ b/samples/bpf/map_perf_test_kern.c @@ -9,25 +9,27 @@ #include #include #include "bpf_helpers.h" +#include "bpf_legacy.h" +#include "bpf_tracing.h" #define MAX_ENTRIES 1000 #define MAX_NR_CPUS 1024 -struct bpf_map_def SEC("maps") hash_map = { +struct bpf_map_def_legacy SEC("maps") hash_map = { .type = BPF_MAP_TYPE_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), .max_entries = MAX_ENTRIES, }; -struct bpf_map_def SEC("maps") lru_hash_map = { +struct bpf_map_def_legacy SEC("maps") lru_hash_map = { .type = BPF_MAP_TYPE_LRU_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), .max_entries = 10000, }; -struct bpf_map_def SEC("maps") nocommon_lru_hash_map = { +struct bpf_map_def_legacy SEC("maps") nocommon_lru_hash_map = { .type = BPF_MAP_TYPE_LRU_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), @@ -35,7 +37,7 @@ struct bpf_map_def SEC("maps") nocommon_lru_hash_map = { .map_flags = BPF_F_NO_COMMON_LRU, }; -struct bpf_map_def SEC("maps") inner_lru_hash_map = { +struct bpf_map_def_legacy SEC("maps") inner_lru_hash_map = { .type = BPF_MAP_TYPE_LRU_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), @@ -44,20 +46,20 @@ struct bpf_map_def SEC("maps") inner_lru_hash_map = { .numa_node = 0, }; -struct bpf_map_def SEC("maps") array_of_lru_hashs = { +struct bpf_map_def_legacy SEC("maps") array_of_lru_hashs = { .type = BPF_MAP_TYPE_ARRAY_OF_MAPS, .key_size = sizeof(u32), .max_entries = MAX_NR_CPUS, }; -struct bpf_map_def SEC("maps") percpu_hash_map = { +struct bpf_map_def_legacy SEC("maps") percpu_hash_map = { .type = BPF_MAP_TYPE_PERCPU_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), .max_entries = MAX_ENTRIES, }; -struct bpf_map_def SEC("maps") hash_map_alloc = { +struct bpf_map_def_legacy SEC("maps") hash_map_alloc = { .type = BPF_MAP_TYPE_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), @@ -65,7 +67,7 @@ struct bpf_map_def SEC("maps") hash_map_alloc = { .map_flags = BPF_F_NO_PREALLOC, }; -struct bpf_map_def SEC("maps") percpu_hash_map_alloc = { +struct bpf_map_def_legacy SEC("maps") percpu_hash_map_alloc = { .type = BPF_MAP_TYPE_PERCPU_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), @@ -73,7 +75,7 @@ struct bpf_map_def SEC("maps") percpu_hash_map_alloc = { .map_flags = BPF_F_NO_PREALLOC, }; -struct bpf_map_def SEC("maps") lpm_trie_map_alloc = { +struct bpf_map_def_legacy SEC("maps") lpm_trie_map_alloc = { .type = BPF_MAP_TYPE_LPM_TRIE, .key_size = 8, .value_size = sizeof(long), @@ -81,14 +83,14 @@ struct bpf_map_def SEC("maps") lpm_trie_map_alloc = { .map_flags = BPF_F_NO_PREALLOC, }; -struct bpf_map_def SEC("maps") array_map = { +struct bpf_map_def_legacy SEC("maps") array_map = { .type = BPF_MAP_TYPE_ARRAY, .key_size = sizeof(u32), .value_size = sizeof(long), .max_entries = MAX_ENTRIES, }; -struct bpf_map_def SEC("maps") lru_hash_lookup_map = { +struct bpf_map_def_legacy SEC("maps") lru_hash_lookup_map = { .type = BPF_MAP_TYPE_LRU_HASH, .key_size = sizeof(u32), .value_size = sizeof(long), diff --git a/samples/bpf/offwaketime_kern.c b/samples/bpf/offwaketime_kern.c index e7d9a0a3d45b12cfa68ace224be590709aba2008..9cb5207a692f294aa9b23edcf4b34d9177f9924f 100644 --- a/samples/bpf/offwaketime_kern.c +++ b/samples/bpf/offwaketime_kern.c @@ -6,6 +6,7 @@ */ #include #include "bpf_helpers.h" +#include "bpf_tracing.h" #include #include #include diff --git a/samples/bpf/parse_ldabs.c b/samples/bpf/parse_ldabs.c index 6db6b21fdc6dd71fd230cfbc40161d087c92024f..ef5892377beba44578f77f168b2d7a4f827b8ca9 100644 --- a/samples/bpf/parse_ldabs.c +++ b/samples/bpf/parse_ldabs.c @@ -12,6 +12,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_legacy.h" #define DEFAULT_PKTGEN_UDP_PORT 9 #define IP_MF 0x2000 diff --git a/samples/bpf/sampleip_kern.c b/samples/bpf/sampleip_kern.c index ceabf31079cf5ef7db360b1182ab64e78d4c05ef..4a190893894f9e69b1be103c053b8c1a513bf1ab 100644 --- a/samples/bpf/sampleip_kern.c +++ b/samples/bpf/sampleip_kern.c @@ -9,6 +9,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" #define MAX_IPS 8192 diff --git a/samples/bpf/sockex1_kern.c b/samples/bpf/sockex1_kern.c index ed18e9a4909c77899d6561680261e97a3dbea235..f96943f443abc0872d0a35dcd30879c8a941efe5 100644 --- a/samples/bpf/sockex1_kern.c +++ b/samples/bpf/sockex1_kern.c @@ -3,6 +3,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_legacy.h" struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_ARRAY, diff --git a/samples/bpf/sockex2_kern.c b/samples/bpf/sockex2_kern.c index f2f9dbc021b0d48c8c66d23cdee444076c25a43d..5566fa7d92fa699d76469e60a5f43713048f9d79 100644 --- a/samples/bpf/sockex2_kern.c +++ b/samples/bpf/sockex2_kern.c @@ -1,5 +1,6 @@ #include #include "bpf_helpers.h" +#include "bpf_legacy.h" #include #include #include diff --git a/samples/bpf/sockex3_kern.c b/samples/bpf/sockex3_kern.c index c527b57d3ec8a5a9aa086d1f8981846173446556..151dd842ecc0b492df8eb8acae495a3420ee3b4b 100644 --- a/samples/bpf/sockex3_kern.c +++ b/samples/bpf/sockex3_kern.c @@ -6,6 +6,7 @@ */ #include #include "bpf_helpers.h" +#include "bpf_legacy.h" #include #include #include diff --git a/samples/bpf/spintest_kern.c b/samples/bpf/spintest_kern.c index ce0167d09cdc6f1432e687933bf38ebef59bad21..6e9478aa2938f40a54b4a1aaff34677e2dcec428 100644 --- a/samples/bpf/spintest_kern.c +++ b/samples/bpf/spintest_kern.c @@ -10,6 +10,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/tcbpf1_kern.c b/samples/bpf/tcbpf1_kern.c index 274c884c87fe01f28adae47feecb9de7f4f0948e..ff43341bdfce12152b236119891fb420786eafde 100644 --- a/samples/bpf/tcbpf1_kern.c +++ b/samples/bpf/tcbpf1_kern.c @@ -8,6 +8,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_legacy.h" /* compiler workaround */ #define _htonl __builtin_bswap32 diff --git a/samples/bpf/test_map_in_map_kern.c b/samples/bpf/test_map_in_map_kern.c index 42c44d091dd176b87bb92d1aeb849b1f8ea8cdd6..4f80cbe74c72bb81b33552cd245a3417590ac6ce 100644 --- a/samples/bpf/test_map_in_map_kern.c +++ b/samples/bpf/test_map_in_map_kern.c @@ -11,11 +11,13 @@ #include #include #include "bpf_helpers.h" +#include "bpf_legacy.h" +#include "bpf_tracing.h" #define MAX_NR_PORTS 65536 /* map #0 */ -struct bpf_map_def SEC("maps") port_a = { +struct bpf_map_def_legacy SEC("maps") port_a = { .type = BPF_MAP_TYPE_ARRAY, .key_size = sizeof(u32), .value_size = sizeof(int), @@ -23,7 +25,7 @@ struct bpf_map_def SEC("maps") port_a = { }; /* map #1 */ -struct bpf_map_def SEC("maps") port_h = { +struct bpf_map_def_legacy SEC("maps") port_h = { .type = BPF_MAP_TYPE_HASH, .key_size = sizeof(u32), .value_size = sizeof(int), @@ -31,7 +33,7 @@ struct bpf_map_def SEC("maps") port_h = { }; /* map #2 */ -struct bpf_map_def SEC("maps") reg_result_h = { +struct bpf_map_def_legacy SEC("maps") reg_result_h = { .type = BPF_MAP_TYPE_HASH, .key_size = sizeof(u32), .value_size = sizeof(int), @@ -39,7 +41,7 @@ struct bpf_map_def SEC("maps") reg_result_h = { }; /* map #3 */ -struct bpf_map_def SEC("maps") inline_result_h = { +struct bpf_map_def_legacy SEC("maps") inline_result_h = { .type = BPF_MAP_TYPE_HASH, .key_size = sizeof(u32), .value_size = sizeof(int), @@ -47,7 +49,7 @@ struct bpf_map_def SEC("maps") inline_result_h = { }; /* map #4 */ /* Test case #0 */ -struct bpf_map_def SEC("maps") a_of_port_a = { +struct bpf_map_def_legacy SEC("maps") a_of_port_a = { .type = BPF_MAP_TYPE_ARRAY_OF_MAPS, .key_size = sizeof(u32), .inner_map_idx = 0, /* map_fd[0] is port_a */ @@ -55,7 +57,7 @@ struct bpf_map_def SEC("maps") a_of_port_a = { }; /* map #5 */ /* Test case #1 */ -struct bpf_map_def SEC("maps") h_of_port_a = { +struct bpf_map_def_legacy SEC("maps") h_of_port_a = { .type = BPF_MAP_TYPE_HASH_OF_MAPS, .key_size = sizeof(u32), .inner_map_idx = 0, /* map_fd[0] is port_a */ @@ -63,7 +65,7 @@ struct bpf_map_def SEC("maps") h_of_port_a = { }; /* map #6 */ /* Test case #2 */ -struct bpf_map_def SEC("maps") h_of_port_h = { +struct bpf_map_def_legacy SEC("maps") h_of_port_h = { .type = BPF_MAP_TYPE_HASH_OF_MAPS, .key_size = sizeof(u32), .inner_map_idx = 1, /* map_fd[1] is port_h */ diff --git a/samples/bpf/test_overhead_kprobe_kern.c b/samples/bpf/test_overhead_kprobe_kern.c index 468a66a92ef9e65816149ebbb6aa168b428344c2..8d2518e68db91530ed406116bc753f39dd22841f 100644 --- a/samples/bpf/test_overhead_kprobe_kern.c +++ b/samples/bpf/test_overhead_kprobe_kern.c @@ -8,6 +8,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" #define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) diff --git a/samples/bpf/test_probe_write_user_kern.c b/samples/bpf/test_probe_write_user_kern.c index 3a677c807044ebefbf1ff4f943109f25822dbbea..a543358218e685d31e61ed43dc633de9b8fd196b 100644 --- a/samples/bpf/test_probe_write_user_kern.c +++ b/samples/bpf/test_probe_write_user_kern.c @@ -9,6 +9,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" struct bpf_map_def SEC("maps") dnat_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/trace_event_kern.c b/samples/bpf/trace_event_kern.c index 7068fbdde951f1ad7b64ff6ba28ff50c382a14bf..8dc18d233a2775d78c11501ce0811f6c3eee0f3b 100644 --- a/samples/bpf/trace_event_kern.c +++ b/samples/bpf/trace_event_kern.c @@ -10,6 +10,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" struct key_t { char comm[TASK_COMM_LEN]; diff --git a/samples/bpf/tracex1_kern.c b/samples/bpf/tracex1_kern.c index 107da148820fc6cdaadcd6e55ab38b8fa78015b5..1a15f6605129e3d9208e48e21c37a4cb8746bf18 100644 --- a/samples/bpf/tracex1_kern.c +++ b/samples/bpf/tracex1_kern.c @@ -9,6 +9,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" #define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) diff --git a/samples/bpf/tracex2_kern.c b/samples/bpf/tracex2_kern.c index 5e11c20ce5ecd9558c4a01f34ad7f994e3418809..d70b3ea79ea70461a6752571085438f86d3413d0 100644 --- a/samples/bpf/tracex2_kern.c +++ b/samples/bpf/tracex2_kern.c @@ -9,6 +9,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/tracex3_kern.c b/samples/bpf/tracex3_kern.c index ea1d4c19c1328e81246c37ac2b523f9964091553..9af546bebfa9a84716b700b74487d7dd343400e6 100644 --- a/samples/bpf/tracex3_kern.c +++ b/samples/bpf/tracex3_kern.c @@ -9,6 +9,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" struct bpf_map_def SEC("maps") my_map = { .type = BPF_MAP_TYPE_HASH, diff --git a/samples/bpf/tracex4_kern.c b/samples/bpf/tracex4_kern.c index 6dd8e384de961b66269f5da840b5d4b22d057ea8..2a02cbe9d9a13a5d3fe05143ed2866b766df8711 100644 --- a/samples/bpf/tracex4_kern.c +++ b/samples/bpf/tracex4_kern.c @@ -8,6 +8,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" struct pair { u64 val; diff --git a/samples/bpf/tracex5_kern.c b/samples/bpf/tracex5_kern.c index 35cb0eed3be591c810235e1f6f623827f70097d1..b3557b21a8fe8711c0e8e5f4c9787ac40ed0e5f3 100644 --- a/samples/bpf/tracex5_kern.c +++ b/samples/bpf/tracex5_kern.c @@ -11,6 +11,7 @@ #include #include "syscall_nrs.h" #include "bpf_helpers.h" +#include "bpf_tracing.h" #define PROG(F) SEC("kprobe/"__stringify(F)) int bpf_func_##F diff --git a/tools/lib/bpf/.gitignore b/tools/lib/bpf/.gitignore index 12382b0c71c7afa9e3e1dc47991951c2089c83d0..35bf013e368c9cde8e2525b13984529a61350a4e 100644 --- a/tools/lib/bpf/.gitignore +++ b/tools/lib/bpf/.gitignore @@ -6,3 +6,4 @@ libbpf.so.* TAGS tags cscope.* +/bpf_helper_defs.h diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index 10b77644a17cfd11f583ec3926a91c5b3e349eb4..1270955e4845c39593f1603faa360773aef6fbb8 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -157,7 +157,7 @@ all: fixdep all_cmd: $(CMD_TARGETS) check -$(BPF_IN): force elfdep bpfdep +$(BPF_IN): force elfdep bpfdep bpf_helper_defs.h @(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \ (diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \ echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true @@ -175,6 +175,10 @@ $(BPF_IN): force elfdep bpfdep echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true $(Q)$(MAKE) $(build)=libbpf +bpf_helper_defs.h: $(srctree)/include/uapi/linux/bpf.h + $(Q)$(srctree)/scripts/bpf_helpers_doc.py --header \ + --file $(srctree)/include/uapi/linux/bpf.h > bpf_helper_defs.h + $(OUTPUT)libbpf.so: $(OUTPUT)libbpf.so.$(LIBBPF_VERSION) $(OUTPUT)libbpf.so.$(LIBBPF_VERSION): $(BPF_IN) @@ -236,13 +240,18 @@ install_lib: all_cmd $(call do_install_mkdir,$(libdir_SQ)); \ cp -fpR $(LIB_FILE) $(DESTDIR)$(libdir_SQ) -install_headers: +install_headers: bpf_helper_defs.h $(call QUIET_INSTALL, headers) \ $(call do_install,bpf.h,$(prefix)/include/bpf,644); \ $(call do_install,libbpf.h,$(prefix)/include/bpf,644); \ $(call do_install,btf.h,$(prefix)/include/bpf,644); \ $(call do_install,libbpf_util.h,$(prefix)/include/bpf,644); \ - $(call do_install,xsk.h,$(prefix)/include/bpf,644); + $(call do_install,xsk.h,$(prefix)/include/bpf,644); \ + $(call do_install,bpf_helpers.h,$(prefix)/include/bpf,644); \ + $(call do_install,bpf_helper_defs.h,$(prefix)/include/bpf,644); \ + $(call do_install,bpf_tracing.h,$(prefix)/include/bpf,644); \ + $(call do_install,bpf_endian.h,$(prefix)/include/bpf,644); \ + $(call do_install,bpf_core_read.h,$(prefix)/include/bpf,644); install_pkgconfig: $(PC_FILE) $(call QUIET_INSTALL, $(PC_FILE)) \ @@ -259,7 +268,7 @@ config-clean: clean: $(call QUIET_CLEAN, libbpf) $(RM) $(TARGETS) $(CXX_TEST_TARGET) \ *.o *~ *.a *.so *.so.$(LIBBPF_MAJOR_VERSION) .*.d .*.cmd \ - *.pc LIBBPF-CFLAGS + *.pc LIBBPF-CFLAGS bpf_helper_defs.h $(call QUIET_CLEAN, core-gen) $(RM) $(OUTPUT)FEATURE-DUMP.libbpf diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h new file mode 100644 index 0000000000000000000000000000000000000000..ae877e3ffb51efe20fb591a06951bb2b9ff74cfb --- /dev/null +++ b/tools/lib/bpf/bpf_core_read.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __BPF_CORE_READ_H__ +#define __BPF_CORE_READ_H__ + +/* + * bpf_core_read() abstracts away bpf_probe_read() call and captures offset + * relocation for source address using __builtin_preserve_access_index() + * built-in, provided by Clang. + * + * __builtin_preserve_access_index() takes as an argument an expression of + * taking an address of a field within struct/union. It makes compiler emit + * a relocation, which records BTF type ID describing root struct/union and an + * accessor string which describes exact embedded field that was used to take + * an address. See detailed description of this relocation format and + * semantics in comments to struct bpf_offset_reloc in libbpf_internal.h. + * + * This relocation allows libbpf to adjust BPF instruction to use correct + * actual field offset, based on target kernel BTF type that matches original + * (local) BTF, used to record relocation. + */ +#define bpf_core_read(dst, sz, src) \ + bpf_probe_read(dst, sz, \ + (const void *)__builtin_preserve_access_index(src)) + +/* + * bpf_core_read_str() is a thin wrapper around bpf_probe_read_str() + * additionally emitting BPF CO-RE field relocation for specified source + * argument. + */ +#define bpf_core_read_str(dst, sz, src) \ + bpf_probe_read_str(dst, sz, \ + (const void *)__builtin_preserve_access_index(src)) + +#define ___concat(a, b) a ## b +#define ___apply(fn, n) ___concat(fn, n) +#define ___nth(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, __11, N, ...) N + +/* + * return number of provided arguments; used for switch-based variadic macro + * definitions (see ___last, ___arrow, etc below) + */ +#define ___narg(...) ___nth(_, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +/* + * return 0 if no arguments are passed, N - otherwise; used for + * recursively-defined macros to specify termination (0) case, and generic + * (N) case (e.g., ___read_ptrs, ___core_read) + */ +#define ___empty(...) ___nth(_, ##__VA_ARGS__, N, N, N, N, N, N, N, N, N, N, 0) + +#define ___last1(x) x +#define ___last2(a, x) x +#define ___last3(a, b, x) x +#define ___last4(a, b, c, x) x +#define ___last5(a, b, c, d, x) x +#define ___last6(a, b, c, d, e, x) x +#define ___last7(a, b, c, d, e, f, x) x +#define ___last8(a, b, c, d, e, f, g, x) x +#define ___last9(a, b, c, d, e, f, g, h, x) x +#define ___last10(a, b, c, d, e, f, g, h, i, x) x +#define ___last(...) ___apply(___last, ___narg(__VA_ARGS__))(__VA_ARGS__) + +#define ___nolast2(a, _) a +#define ___nolast3(a, b, _) a, b +#define ___nolast4(a, b, c, _) a, b, c +#define ___nolast5(a, b, c, d, _) a, b, c, d +#define ___nolast6(a, b, c, d, e, _) a, b, c, d, e +#define ___nolast7(a, b, c, d, e, f, _) a, b, c, d, e, f +#define ___nolast8(a, b, c, d, e, f, g, _) a, b, c, d, e, f, g +#define ___nolast9(a, b, c, d, e, f, g, h, _) a, b, c, d, e, f, g, h +#define ___nolast10(a, b, c, d, e, f, g, h, i, _) a, b, c, d, e, f, g, h, i +#define ___nolast(...) ___apply(___nolast, ___narg(__VA_ARGS__))(__VA_ARGS__) + +#define ___arrow1(a) a +#define ___arrow2(a, b) a->b +#define ___arrow3(a, b, c) a->b->c +#define ___arrow4(a, b, c, d) a->b->c->d +#define ___arrow5(a, b, c, d, e) a->b->c->d->e +#define ___arrow6(a, b, c, d, e, f) a->b->c->d->e->f +#define ___arrow7(a, b, c, d, e, f, g) a->b->c->d->e->f->g +#define ___arrow8(a, b, c, d, e, f, g, h) a->b->c->d->e->f->g->h +#define ___arrow9(a, b, c, d, e, f, g, h, i) a->b->c->d->e->f->g->h->i +#define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j +#define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__) + +#define ___type(...) typeof(___arrow(__VA_ARGS__)) + +#define ___read(read_fn, dst, src_type, src, accessor) \ + read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor) + +/* "recursively" read a sequence of inner pointers using local __t var */ +#define ___rd_last(...) \ + ___read(bpf_core_read, &__t, \ + ___type(___nolast(__VA_ARGS__)), __t, ___last(__VA_ARGS__)); +#define ___rd_p0(src) const void *__t = src; +#define ___rd_p1(...) ___rd_p0(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p2(...) ___rd_p1(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p3(...) ___rd_p2(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p4(...) ___rd_p3(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p5(...) ___rd_p4(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p6(...) ___rd_p5(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p7(...) ___rd_p6(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p8(...) ___rd_p7(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___rd_p9(...) ___rd_p8(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) +#define ___read_ptrs(src, ...) \ + ___apply(___rd_p, ___narg(__VA_ARGS__))(src, __VA_ARGS__) + +#define ___core_read0(fn, dst, src, a) \ + ___read(fn, dst, ___type(src), src, a); +#define ___core_readN(fn, dst, src, ...) \ + ___read_ptrs(src, ___nolast(__VA_ARGS__)) \ + ___read(fn, dst, ___type(src, ___nolast(__VA_ARGS__)), __t, \ + ___last(__VA_ARGS__)); +#define ___core_read(fn, dst, src, a, ...) \ + ___apply(___core_read, ___empty(__VA_ARGS__))(fn, dst, \ + src, a, ##__VA_ARGS__) + +/* + * BPF_CORE_READ_INTO() is a more performance-conscious variant of + * BPF_CORE_READ(), in which final field is read into user-provided storage. + * See BPF_CORE_READ() below for more details on general usage. + */ +#define BPF_CORE_READ_INTO(dst, src, a, ...) \ + ({ \ + ___core_read(bpf_core_read, dst, src, a, ##__VA_ARGS__) \ + }) + +/* + * BPF_CORE_READ_STR_INTO() does same "pointer chasing" as + * BPF_CORE_READ() for intermediate pointers, but then executes (and returns + * corresponding error code) bpf_core_read_str() for final string read. + */ +#define BPF_CORE_READ_STR_INTO(dst, src, a, ...) \ + ({ \ + ___core_read(bpf_core_read_str, dst, src, a, ##__VA_ARGS__) \ + }) + +/* + * BPF_CORE_READ() is used to simplify BPF CO-RE relocatable read, especially + * when there are few pointer chasing steps. + * E.g., what in non-BPF world (or in BPF w/ BCC) would be something like: + * int x = s->a.b.c->d.e->f->g; + * can be succinctly achieved using BPF_CORE_READ as: + * int x = BPF_CORE_READ(s, a.b.c, d.e, f, g); + * + * BPF_CORE_READ will decompose above statement into 4 bpf_core_read (BPF + * CO-RE relocatable bpf_probe_read() wrapper) calls, logically equivalent to: + * 1. const void *__t = s->a.b.c; + * 2. __t = __t->d.e; + * 3. __t = __t->f; + * 4. return __t->g; + * + * Equivalence is logical, because there is a heavy type casting/preservation + * involved, as well as all the reads are happening through bpf_probe_read() + * calls using __builtin_preserve_access_index() to emit CO-RE relocations. + * + * N.B. Only up to 9 "field accessors" are supported, which should be more + * than enough for any practical purpose. + */ +#define BPF_CORE_READ(src, a, ...) \ + ({ \ + ___type(src, a, ##__VA_ARGS__) __r; \ + BPF_CORE_READ_INTO(&__r, src, a, ##__VA_ARGS__); \ + __r; \ + }) + +#endif + diff --git a/tools/testing/selftests/bpf/bpf_endian.h b/tools/lib/bpf/bpf_endian.h similarity index 100% rename from tools/testing/selftests/bpf/bpf_endian.h rename to tools/lib/bpf/bpf_endian.h diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h new file mode 100644 index 0000000000000000000000000000000000000000..2203595f38c394fb0cc9776b7f868962a2b51faf --- /dev/null +++ b/tools/lib/bpf/bpf_helpers.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __BPF_HELPERS__ +#define __BPF_HELPERS__ + +#include "bpf_helper_defs.h" + +#define __uint(name, val) int (*name)[val] +#define __type(name, val) typeof(val) *name + +/* Helper macro to print out debug messages */ +#define bpf_printk(fmt, ...) \ +({ \ + char ____fmt[] = fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), \ + ##__VA_ARGS__); \ +}) + +/* + * Helper macro to place programs, maps, license in + * different sections in elf_bpf file. Section names + * are interpreted by elf_bpf loader + */ +#define SEC(NAME) __attribute__((section(NAME), used)) + +#ifndef __always_inline +#define __always_inline __attribute__((always_inline)) +#endif + +/* + * Helper structure used by eBPF C program + * to describe BPF map attributes to libbpf loader + */ +struct bpf_map_def { + unsigned int type; + unsigned int key_size; + unsigned int value_size; + unsigned int max_entries; + unsigned int map_flags; +}; + +#endif diff --git a/tools/testing/selftests/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_tracing.h similarity index 68% rename from tools/testing/selftests/bpf/bpf_helpers.h rename to tools/lib/bpf/bpf_tracing.h index 15152280db6f1d4a38c6280cc1a36c6db2e6e17b..b0dafe8b4ebc0dbea54613db07cab037ab103a6a 100644 --- a/tools/testing/selftests/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -1,66 +1,6 @@ /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __BPF_HELPERS__ -#define __BPF_HELPERS__ - -#include "bpf_helper_defs.h" - -#define __uint(name, val) int (*name)[val] -#define __type(name, val) typeof(val) *name - -/* helper macro to print out debug messages */ -#define bpf_printk(fmt, ...) \ -({ \ - char ____fmt[] = fmt; \ - bpf_trace_printk(____fmt, sizeof(____fmt), \ - ##__VA_ARGS__); \ -}) - -#ifdef __clang__ - -/* helper macro to place programs, maps, license in - * different sections in elf_bpf file. Section names - * are interpreted by elf_bpf loader - */ -#define SEC(NAME) __attribute__((section(NAME), used)) - -/* llvm builtin functions that eBPF C program may use to - * emit BPF_LD_ABS and BPF_LD_IND instructions - */ -struct sk_buff; -unsigned long long load_byte(void *skb, - unsigned long long off) asm("llvm.bpf.load.byte"); -unsigned long long load_half(void *skb, - unsigned long long off) asm("llvm.bpf.load.half"); -unsigned long long load_word(void *skb, - unsigned long long off) asm("llvm.bpf.load.word"); - -/* a helper structure used by eBPF C program - * to describe map attributes to elf_bpf loader - */ -struct bpf_map_def { - unsigned int type; - unsigned int key_size; - unsigned int value_size; - unsigned int max_entries; - unsigned int map_flags; - unsigned int inner_map_idx; - unsigned int numa_node; -}; - -#else - -#include - -#endif - -#define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ - struct ____btf_map_##name { \ - type_key key; \ - type_val value; \ - }; \ - struct ____btf_map_##name \ - __attribute__ ((section(".maps." #name), used)) \ - ____btf_map_##name = { } +#ifndef __BPF_TRACING_H__ +#define __BPF_TRACING_H__ /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ #if defined(__TARGET_ARCH_x86) @@ -245,31 +185,11 @@ struct pt_regs; #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP #else -#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ \ - bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) -#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ \ - bpf_probe_read(&(ip), sizeof(ip), \ - (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) +#define BPF_KPROBE_READ_RET_IP(ip, ctx) \ + ({ bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) +#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ + ({ bpf_probe_read(&(ip), sizeof(ip), \ + (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) #endif -/* - * BPF_CORE_READ abstracts away bpf_probe_read() call and captures offset - * relocation for source address using __builtin_preserve_access_index() - * built-in, provided by Clang. - * - * __builtin_preserve_access_index() takes as an argument an expression of - * taking an address of a field within struct/union. It makes compiler emit - * a relocation, which records BTF type ID describing root struct/union and an - * accessor string which describes exact embedded field that was used to take - * an address. See detailed description of this relocation format and - * semantics in comments to struct bpf_offset_reloc in libbpf_internal.h. - * - * This relocation allows libbpf to adjust BPF instruction to use correct - * actual field offset, based on target kernel BTF type that matches original - * (local) BTF, used to record relocation. - */ -#define BPF_CORE_READ(dst, src) \ - bpf_probe_read((dst), sizeof(*(src)), \ - __builtin_preserve_access_index(src)) - #endif diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index 50063f66539d512c52e512a96db1fcd9fbcb7fa3..7470327edcfec2717b8f4c2d619402f91142efb2 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -39,4 +39,3 @@ libbpf.so.* test_hashmap test_btf_dump xdping -/bpf_helper_defs.h diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 771a4e82128b1af2625e5db0925c2a345f6d4ed8..90944b7a8274c544f293912dc1b1714d91deb6bb 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -90,10 +90,6 @@ include ../lib.mk TEST_CUSTOM_PROGS = $(OUTPUT)/urandom_read all: $(TEST_CUSTOM_PROGS) -bpf_helper_defs.h: $(APIDIR)/linux/bpf.h - $(BPFDIR)/../../../scripts/bpf_helpers_doc.py --header \ - --file $(APIDIR)/linux/bpf.h > bpf_helper_defs.h - $(OUTPUT)/urandom_read: $(OUTPUT)/%: %.c $(CC) -o $@ $< -Wl,--build-id @@ -127,7 +123,7 @@ $(OUTPUT)/test_cgroup_attach: cgroup_helpers.c # force a rebuild of BPFOBJ when its dependencies are updated force: -$(BPFOBJ): force bpf_helper_defs.h +$(BPFOBJ): force $(MAKE) -C $(BPFDIR) OUTPUT=$(OUTPUT)/ PROBE := $(shell $(LLC) -march=bpf -mcpu=probe -filetype=null /dev/null 2>&1) @@ -152,7 +148,7 @@ $(shell $(1) -v -E - &1 \ endef CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG)) BPF_CFLAGS = -I. -I./include/uapi -I../../../include/uapi \ - -I$(OUTPUT)/../usr/include -D__TARGET_ARCH_$(SRCARCH) + -I$(BPFDIR) -I$(OUTPUT)/../usr/include -D__TARGET_ARCH_$(SRCARCH) CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \ -Wno-compare-distinct-pointer-types @@ -323,4 +319,4 @@ $(VERIFIER_TESTS_H): $(VERIFIER_TEST_FILES) | $(VERIFIER_TESTS_DIR) EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(ALU32_BUILD_DIR) $(BPF_GCC_BUILD_DIR) \ $(VERIFIER_TESTS_H) $(PROG_TESTS_H) $(MAP_TESTS_H) \ - feature bpf_helper_defs.h + feature diff --git a/tools/testing/selftests/bpf/bpf_legacy.h b/tools/testing/selftests/bpf/bpf_legacy.h new file mode 100644 index 0000000000000000000000000000000000000000..6f8988738bc15c56f4b58679338ec784acaedffb --- /dev/null +++ b/tools/testing/selftests/bpf/bpf_legacy.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __BPF_LEGACY__ +#define __BPF_LEGACY__ + +/* + * legacy bpf_map_def with extra fields supported only by bpf_load(), do not + * use outside of samples/bpf + */ +struct bpf_map_def_legacy { + unsigned int type; + unsigned int key_size; + unsigned int value_size; + unsigned int max_entries; + unsigned int map_flags; + unsigned int inner_map_idx; + unsigned int numa_node; +}; + +#define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ + struct ____btf_map_##name { \ + type_key key; \ + type_val value; \ + }; \ + struct ____btf_map_##name \ + __attribute__ ((section(".maps." #name), used)) \ + ____btf_map_##name = { } + +/* llvm builtin functions that eBPF C program may use to + * emit BPF_LD_ABS and BPF_LD_IND instructions + */ +unsigned long long load_byte(void *skb, + unsigned long long off) asm("llvm.bpf.load.byte"); +unsigned long long load_half(void *skb, + unsigned long long off) asm("llvm.bpf.load.half"); +unsigned long long load_word(void *skb, + unsigned long long off) asm("llvm.bpf.load.word"); + +#endif + diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index f3863f976a48e337032c0a522be0ad7eb80cd54e..21a0dff662412fbdf8bc6b3e31a8b8b7bd72e497 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -193,8 +193,12 @@ static struct core_reloc_test_case test_cases[] = { .btf_src_file = NULL, /* load from /lib/modules/$(uname -r) */ .input = "", .input_len = 0, - .output = "\1", /* true */ - .output_len = 1, + .output = STRUCT_TO_CHAR_PTR(core_reloc_kernel_output) { + .valid = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }, + .comm = "test_progs\0\0\0\0\0", + .comm_len = 11, + }, + .output_len = sizeof(struct core_reloc_kernel_output), }, /* validate BPF program can use multiple flavors to match against diff --git a/tools/testing/selftests/bpf/progs/core_reloc_types.h b/tools/testing/selftests/bpf/progs/core_reloc_types.h index f686a8138d90a26d630afa4a61d44dce457ae24a..9a6bdeb4894c4201ee91862504a04df0af375cac 100644 --- a/tools/testing/selftests/bpf/progs/core_reloc_types.h +++ b/tools/testing/selftests/bpf/progs/core_reloc_types.h @@ -1,5 +1,14 @@ #include #include +/* + * KERNEL + */ + +struct core_reloc_kernel_output { + int valid[10]; + char comm[16]; + int comm_len; +}; /* * FLAVORS diff --git a/tools/testing/selftests/bpf/progs/loop1.c b/tools/testing/selftests/bpf/progs/loop1.c index 7cdb7f878310b2dba828432d421ac61e0c69dfae..40ac722a9da5e850363af912a264b2b6de4abdb3 100644 --- a/tools/testing/selftests/bpf/progs/loop1.c +++ b/tools/testing/selftests/bpf/progs/loop1.c @@ -7,6 +7,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/loop2.c b/tools/testing/selftests/bpf/progs/loop2.c index 9b2f808a2863732f7a3648ed6a875c1a93b57425..bb80f29aa7f7fd746a1ea460227e8347f52975bd 100644 --- a/tools/testing/selftests/bpf/progs/loop2.c +++ b/tools/testing/selftests/bpf/progs/loop2.c @@ -7,6 +7,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/loop3.c b/tools/testing/selftests/bpf/progs/loop3.c index d727657d51e2ab95afe97465f2ba9a63ce7bc23c..2b9165a7afe1903b6daebb2e9b8b30bc63c00249 100644 --- a/tools/testing/selftests/bpf/progs/loop3.c +++ b/tools/testing/selftests/bpf/progs/loop3.c @@ -7,6 +7,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_tracing.h" char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/sockopt_sk.c b/tools/testing/selftests/bpf/progs/sockopt_sk.c index 9a3d1c79e6fe980c426c918df343733e1d397f25..1bafbb944e37405191416406fe878ac2dffe5956 100644 --- a/tools/testing/selftests/bpf/progs/sockopt_sk.c +++ b/tools/testing/selftests/bpf/progs/sockopt_sk.c @@ -14,13 +14,12 @@ struct sockopt_sk { __u8 val; }; -struct bpf_map_def SEC("maps") socket_storage_map = { - .type = BPF_MAP_TYPE_SK_STORAGE, - .key_size = sizeof(int), - .value_size = sizeof(struct sockopt_sk), - .map_flags = BPF_F_NO_PREALLOC, -}; -BPF_ANNOTATE_KV_PAIR(socket_storage_map, int, struct sockopt_sk); +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct sockopt_sk); +} socket_storage_map SEC(".maps"); SEC("cgroup/getsockopt") int _getsockopt(struct bpf_sockopt *ctx) diff --git a/tools/testing/selftests/bpf/progs/tcp_rtt.c b/tools/testing/selftests/bpf/progs/tcp_rtt.c index 233bdcb1659ef7ab3ce9a3be7d8745a865529f0d..2cf813a06b836245e97322728c8951aca36a9405 100644 --- a/tools/testing/selftests/bpf/progs/tcp_rtt.c +++ b/tools/testing/selftests/bpf/progs/tcp_rtt.c @@ -13,13 +13,12 @@ struct tcp_rtt_storage { __u32 icsk_retransmits; }; -struct bpf_map_def SEC("maps") socket_storage_map = { - .type = BPF_MAP_TYPE_SK_STORAGE, - .key_size = sizeof(int), - .value_size = sizeof(struct tcp_rtt_storage), - .map_flags = BPF_F_NO_PREALLOC, -}; -BPF_ANNOTATE_KV_PAIR(socket_storage_map, int, struct tcp_rtt_storage); +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct tcp_rtt_storage); +} socket_storage_map SEC(".maps"); SEC("sockops") int _sockops(struct bpf_sock_ops *ctx) diff --git a/tools/testing/selftests/bpf/progs/test_btf_haskv.c b/tools/testing/selftests/bpf/progs/test_btf_haskv.c index e5c79fe0ffdb22f5ee797c3f1d512452613587f4..763c51447c19b3fe949600e789249f54d5dceb8a 100644 --- a/tools/testing/selftests/bpf/progs/test_btf_haskv.c +++ b/tools/testing/selftests/bpf/progs/test_btf_haskv.c @@ -2,6 +2,7 @@ /* Copyright (c) 2018 Facebook */ #include #include "bpf_helpers.h" +#include "bpf_legacy.h" int _version SEC("version") = 1; diff --git a/tools/testing/selftests/bpf/progs/test_btf_newkv.c b/tools/testing/selftests/bpf/progs/test_btf_newkv.c index 5ee3622ddebb698d6461ab400e3a8685cee523b6..96f9e84510293c34cc5a4812c3358f9821cb3d45 100644 --- a/tools/testing/selftests/bpf/progs/test_btf_newkv.c +++ b/tools/testing/selftests/bpf/progs/test_btf_newkv.c @@ -2,6 +2,7 @@ /* Copyright (c) 2018 Facebook */ #include #include "bpf_helpers.h" +#include "bpf_legacy.h" int _version SEC("version") = 1; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c b/tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c index bf67f0fdf743e329ce406ead206954001f2ec71c..96b1f5f3b07a2b23763d5d112db45b962b24979e 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -31,6 +32,8 @@ struct core_reloc_arrays { struct core_reloc_arrays_substruct d[1][2]; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_arrays(void *ctx) { @@ -38,16 +41,16 @@ int test_core_arrays(void *ctx) struct core_reloc_arrays_output *out = (void *)&data.out; /* in->a[2] */ - if (BPF_CORE_READ(&out->a2, &in->a[2])) + if (CORE_READ(&out->a2, &in->a[2])) return 1; /* in->b[1][2][3] */ - if (BPF_CORE_READ(&out->b123, &in->b[1][2][3])) + if (CORE_READ(&out->b123, &in->b[1][2][3])) return 1; /* in->c[1].c */ - if (BPF_CORE_READ(&out->c1c, &in->c[1].c)) + if (CORE_READ(&out->c1c, &in->c[1].c)) return 1; /* in->d[0][0].d */ - if (BPF_CORE_READ(&out->d00d, &in->d[0][0].d)) + if (CORE_READ(&out->d00d, &in->d[0][0].d)) return 1; return 0; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_flavors.c b/tools/testing/selftests/bpf/progs/test_core_reloc_flavors.c index 9fda73e879727a6aafc3b6d81f41f8debb6e8c69..71fd7cebc9d7907bfb02b6bcc40754954b5e8518 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_flavors.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_flavors.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -39,6 +40,8 @@ struct core_reloc_flavors___weird { }; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_flavors(void *ctx) { @@ -48,13 +51,13 @@ int test_core_flavors(void *ctx) struct core_reloc_flavors *out = (void *)&data.out; /* read a using weird layout */ - if (BPF_CORE_READ(&out->a, &in_weird->a)) + if (CORE_READ(&out->a, &in_weird->a)) return 1; /* read b using reversed layout */ - if (BPF_CORE_READ(&out->b, &in_rev->b)) + if (CORE_READ(&out->b, &in_rev->b)) return 1; /* read c using original layout */ - if (BPF_CORE_READ(&out->c, &in_orig->c)) + if (CORE_READ(&out->c, &in_orig->c)) return 1; return 0; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_ints.c b/tools/testing/selftests/bpf/progs/test_core_reloc_ints.c index d99233c8008aacdc68d4b7c5f0290476c7bd0a95..ad5c3f59c9c69431737de47eeebab06cc75112f0 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_ints.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_ints.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -23,20 +24,22 @@ struct core_reloc_ints { int64_t s64_field; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_ints(void *ctx) { struct core_reloc_ints *in = (void *)&data.in; struct core_reloc_ints *out = (void *)&data.out; - if (BPF_CORE_READ(&out->u8_field, &in->u8_field) || - BPF_CORE_READ(&out->s8_field, &in->s8_field) || - BPF_CORE_READ(&out->u16_field, &in->u16_field) || - BPF_CORE_READ(&out->s16_field, &in->s16_field) || - BPF_CORE_READ(&out->u32_field, &in->u32_field) || - BPF_CORE_READ(&out->s32_field, &in->s32_field) || - BPF_CORE_READ(&out->u64_field, &in->u64_field) || - BPF_CORE_READ(&out->s64_field, &in->s64_field)) + if (CORE_READ(&out->u8_field, &in->u8_field) || + CORE_READ(&out->s8_field, &in->s8_field) || + CORE_READ(&out->u16_field, &in->u16_field) || + CORE_READ(&out->s16_field, &in->s16_field) || + CORE_READ(&out->u32_field, &in->u32_field) || + CORE_READ(&out->s32_field, &in->s32_field) || + CORE_READ(&out->u64_field, &in->u64_field) || + CORE_READ(&out->s64_field, &in->s64_field)) return 1; return 0; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c b/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c index 37e02aa3f0c8b3e3a5908df7f2d2aa4fe7e9cf2d..50f609618b65f443e52ea2fec71f3d88062acaf7 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -12,24 +13,78 @@ static volatile struct data { char out[256]; } data; +struct core_reloc_kernel_output { + int valid[10]; + char comm[16]; + int comm_len; +}; + struct task_struct { int pid; int tgid; + char comm[16]; + struct task_struct *group_leader; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_kernel(void *ctx) { struct task_struct *task = (void *)bpf_get_current_task(); + struct core_reloc_kernel_output *out = (void *)&data.out; uint64_t pid_tgid = bpf_get_current_pid_tgid(); + uint32_t real_tgid = (uint32_t)pid_tgid; int pid, tgid; - if (BPF_CORE_READ(&pid, &task->pid) || - BPF_CORE_READ(&tgid, &task->tgid)) + if (CORE_READ(&pid, &task->pid) || + CORE_READ(&tgid, &task->tgid)) return 1; /* validate pid + tgid matches */ - data.out[0] = (((uint64_t)pid << 32) | tgid) == pid_tgid; + out->valid[0] = (((uint64_t)pid << 32) | tgid) == pid_tgid; + + /* test variadic BPF_CORE_READ macros */ + out->valid[1] = BPF_CORE_READ(task, + tgid) == real_tgid; + out->valid[2] = BPF_CORE_READ(task, + group_leader, + tgid) == real_tgid; + out->valid[3] = BPF_CORE_READ(task, + group_leader, group_leader, + tgid) == real_tgid; + out->valid[4] = BPF_CORE_READ(task, + group_leader, group_leader, group_leader, + tgid) == real_tgid; + out->valid[5] = BPF_CORE_READ(task, + group_leader, group_leader, group_leader, + group_leader, + tgid) == real_tgid; + out->valid[6] = BPF_CORE_READ(task, + group_leader, group_leader, group_leader, + group_leader, group_leader, + tgid) == real_tgid; + out->valid[7] = BPF_CORE_READ(task, + group_leader, group_leader, group_leader, + group_leader, group_leader, group_leader, + tgid) == real_tgid; + out->valid[8] = BPF_CORE_READ(task, + group_leader, group_leader, group_leader, + group_leader, group_leader, group_leader, + group_leader, + tgid) == real_tgid; + out->valid[9] = BPF_CORE_READ(task, + group_leader, group_leader, group_leader, + group_leader, group_leader, group_leader, + group_leader, group_leader, + tgid) == real_tgid; + + /* test BPF_CORE_READ_STR_INTO() returns correct code and contents */ + out->comm_len = BPF_CORE_READ_STR_INTO( + &out->comm, task, + group_leader, group_leader, group_leader, group_leader, + group_leader, group_leader, group_leader, group_leader, + comm); return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_misc.c b/tools/testing/selftests/bpf/progs/test_core_reloc_misc.c index c59984bd3e235e7552663c6c4d7a2bee1c4ee789..1a36b0856653851f34583f50561357fd88bd5d18 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_misc.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_misc.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -32,6 +33,8 @@ struct core_reloc_misc_extensible { int b; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_misc(void *ctx) { @@ -41,15 +44,15 @@ int test_core_misc(void *ctx) struct core_reloc_misc_output *out = (void *)&data.out; /* record two different relocations with the same accessor string */ - if (BPF_CORE_READ(&out->a, &in_a->a1) || /* accessor: 0:0 */ - BPF_CORE_READ(&out->b, &in_b->b1)) /* accessor: 0:0 */ + if (CORE_READ(&out->a, &in_a->a1) || /* accessor: 0:0 */ + CORE_READ(&out->b, &in_b->b1)) /* accessor: 0:0 */ return 1; /* Validate relocations capture array-only accesses for structs with * fixed header, but with potentially extendable tail. This will read * first 4 bytes of 2nd element of in_ext array of potentially * variably sized struct core_reloc_misc_extensible. */ - if (BPF_CORE_READ(&out->c, &in_ext[2])) /* accessor: 2 */ + if (CORE_READ(&out->c, &in_ext[2])) /* accessor: 2 */ return 1; return 0; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_mods.c b/tools/testing/selftests/bpf/progs/test_core_reloc_mods.c index f98b942c062ba2f5cd09fd20ba059beae9e3fa00..3199fafede2cd4dab3daa9013a259a82529ae339 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_mods.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_mods.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -41,20 +42,22 @@ struct core_reloc_mods { core_reloc_mods_substruct_t h; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_mods(void *ctx) { struct core_reloc_mods *in = (void *)&data.in; struct core_reloc_mods_output *out = (void *)&data.out; - if (BPF_CORE_READ(&out->a, &in->a) || - BPF_CORE_READ(&out->b, &in->b) || - BPF_CORE_READ(&out->c, &in->c) || - BPF_CORE_READ(&out->d, &in->d) || - BPF_CORE_READ(&out->e, &in->e[2]) || - BPF_CORE_READ(&out->f, &in->f[1]) || - BPF_CORE_READ(&out->g, &in->g.x) || - BPF_CORE_READ(&out->h, &in->h.y)) + if (CORE_READ(&out->a, &in->a) || + CORE_READ(&out->b, &in->b) || + CORE_READ(&out->c, &in->c) || + CORE_READ(&out->d, &in->d) || + CORE_READ(&out->e, &in->e[2]) || + CORE_READ(&out->f, &in->f[1]) || + CORE_READ(&out->g, &in->g.x) || + CORE_READ(&out->h, &in->h.y)) return 1; return 0; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_nesting.c b/tools/testing/selftests/bpf/progs/test_core_reloc_nesting.c index 3ca30cec2b390474291f98012241e9648c849632..98238cb64fbd74b964924a8c68a34f37d06fb7b9 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_nesting.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_nesting.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -30,15 +31,17 @@ struct core_reloc_nesting { } b; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_nesting(void *ctx) { struct core_reloc_nesting *in = (void *)&data.in; struct core_reloc_nesting *out = (void *)&data.out; - if (BPF_CORE_READ(&out->a.a.a, &in->a.a.a)) + if (CORE_READ(&out->a.a.a, &in->a.a.a)) return 1; - if (BPF_CORE_READ(&out->b.b.b, &in->b.b.b)) + if (CORE_READ(&out->b.b.b, &in->b.b.b)) return 1; return 0; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_primitives.c b/tools/testing/selftests/bpf/progs/test_core_reloc_primitives.c index add52f23ab35237576ccaabed1b14156ac27c97b..4f3ecb9127bb24f159fda18c0bd0d919a2ceda09 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_primitives.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_primitives.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -25,17 +26,19 @@ struct core_reloc_primitives { int (*f)(const char *); }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_primitives(void *ctx) { struct core_reloc_primitives *in = (void *)&data.in; struct core_reloc_primitives *out = (void *)&data.out; - if (BPF_CORE_READ(&out->a, &in->a) || - BPF_CORE_READ(&out->b, &in->b) || - BPF_CORE_READ(&out->c, &in->c) || - BPF_CORE_READ(&out->d, &in->d) || - BPF_CORE_READ(&out->f, &in->f)) + if (CORE_READ(&out->a, &in->a) || + CORE_READ(&out->b, &in->b) || + CORE_READ(&out->c, &in->c) || + CORE_READ(&out->d, &in->d) || + CORE_READ(&out->f, &in->f)) return 1; return 0; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_ptr_as_arr.c b/tools/testing/selftests/bpf/progs/test_core_reloc_ptr_as_arr.c index 526b7ddc7ea1d50c464558182549711eb925bf81..27f602f00419357a158c06c39b799f09416fd0a2 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_ptr_as_arr.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_ptr_as_arr.c @@ -4,6 +4,7 @@ #include #include #include "bpf_helpers.h" +#include "bpf_core_read.h" char _license[] SEC("license") = "GPL"; @@ -16,13 +17,15 @@ struct core_reloc_ptr_as_arr { int a; }; +#define CORE_READ(dst, src) bpf_core_read(dst, sizeof(*(dst)), src) + SEC("raw_tracepoint/sys_enter") int test_core_ptr_as_arr(void *ctx) { struct core_reloc_ptr_as_arr *in = (void *)&data.in; struct core_reloc_ptr_as_arr *out = (void *)&data.out; - if (BPF_CORE_READ(&out->a, &in[2].a)) + if (CORE_READ(&out->a, &in[2].a)) return 1; return 0;