108 bool bpf_prog_has_trampoline(const struct bpf_prog *prog)  in bpf_prog_has_trampoline()  argument
110 	enum bpf_attach_type eatype = prog->expected_attach_type;  in bpf_prog_has_trampoline()
111 	enum bpf_prog_type ptype = prog->type;  in bpf_prog_has_trampoline()
275 			*ip_arg |= link->link.prog->call_get_func_ip;  in bpf_trampoline_get_progs()
514 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)  in bpf_attach_type_to_tramp()  argument
516 	switch (prog->expected_attach_type) {  in bpf_attach_type_to_tramp()
524 		if (!prog->aux->attach_func_proto->type)  in bpf_attach_type_to_tramp()
543 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_link_prog()
557 		tr->extension_prog = link->link.prog;  in __bpf_trampoline_link_prog()
559 					  link->link.prog->bpf_func);  in __bpf_trampoline_link_prog()
567 		if (link_exiting->link.prog != link->link.prog)  in __bpf_trampoline_link_prog()
598 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_unlink_prog()
649 static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,  in cgroup_shim_alloc()  argument
670 	p->aux->attach_func_proto = prog->aux->attach_func_proto;  in cgroup_shim_alloc()
671 	p->aux->attach_btf_id = prog->aux->attach_btf_id;  in cgroup_shim_alloc()
672 	p->aux->attach_btf = prog->aux->attach_btf;  in cgroup_shim_alloc()
692 			struct bpf_prog *p = link->link.prog;  in cgroup_shim_find()
702 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,  in bpf_trampoline_link_cgroup_shim()  argument
712 	err = bpf_check_attach_target(NULL, prog, NULL,  in bpf_trampoline_link_cgroup_shim()
713 				      prog->aux->attach_btf_id,  in bpf_trampoline_link_cgroup_shim()
718 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_link_cgroup_shim()
719 					 prog->aux->attach_btf_id);  in bpf_trampoline_link_cgroup_shim()
721 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_link_cgroup_shim()
740 	shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);  in bpf_trampoline_link_cgroup_shim()
768 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)  in bpf_trampoline_unlink_cgroup_shim()  argument
775 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_unlink_cgroup_shim()
776 					 prog->aux->attach_btf_id);  in bpf_trampoline_unlink_cgroup_shim()
778 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_unlink_cgroup_shim()
871 u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_enter()  argument
879 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter()
880 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter()
886 static void notrace update_prog_stats(struct bpf_prog *prog,  in update_prog_stats()  argument
900 		stats = this_cpu_ptr(prog->stats);  in update_prog_stats()
908 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_exit()  argument
913 	update_prog_stats(prog, start);  in __bpf_prog_exit()
914 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit()
919 u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,  in __bpf_prog_enter_lsm_cgroup()  argument
934 void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_lsm_cgroup()  argument
944 u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_enter_sleepable()  argument
950 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_sleepable()
951 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_sleepable()
960 void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable()  argument
965 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable()
966 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_sleepable()
971 u64 notrace __bpf_prog_enter_struct_ops(struct bpf_prog *prog,  in __bpf_prog_enter_struct_ops()  argument
983 void notrace __bpf_prog_exit_struct_ops(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_struct_ops()  argument
989 	update_prog_stats(prog, start);  in __bpf_prog_exit_struct_ops()