109 	fp->aux->prog = fp;  in bpf_prog_alloc_no_stats()
126 	struct bpf_prog *prog;  in bpf_prog_alloc()  local
129 	prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags);  in bpf_prog_alloc()
130 	if (!prog)  in bpf_prog_alloc()
133 	prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);  in bpf_prog_alloc()
134 	if (!prog->stats) {  in bpf_prog_alloc()
135 		free_percpu(prog->active);  in bpf_prog_alloc()
136 		kfree(prog->aux);  in bpf_prog_alloc()
137 		vfree(prog);  in bpf_prog_alloc()
144 		pstats = per_cpu_ptr(prog->stats, cpu);  in bpf_prog_alloc()
147 	return prog;  in bpf_prog_alloc()
151 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)  in bpf_prog_alloc_jited_linfo()  argument
153 	if (!prog->aux->nr_linfo || !prog->jit_requested)  in bpf_prog_alloc_jited_linfo()
156 	prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,  in bpf_prog_alloc_jited_linfo()
157 					  sizeof(*prog->aux->jited_linfo),  in bpf_prog_alloc_jited_linfo()
159 	if (!prog->aux->jited_linfo)  in bpf_prog_alloc_jited_linfo()
165 void bpf_prog_jit_attempt_done(struct bpf_prog *prog)  in bpf_prog_jit_attempt_done()  argument
167 	if (prog->aux->jited_linfo &&  in bpf_prog_jit_attempt_done()
168 	    (!prog->jited || !prog->aux->jited_linfo[0])) {  in bpf_prog_jit_attempt_done()
169 		kvfree(prog->aux->jited_linfo);  in bpf_prog_jit_attempt_done()
170 		prog->aux->jited_linfo = NULL;  in bpf_prog_jit_attempt_done()
173 	kfree(prog->aux->kfunc_tab);  in bpf_prog_jit_attempt_done()
174 	prog->aux->kfunc_tab = NULL;  in bpf_prog_jit_attempt_done()
201 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,  in bpf_prog_fill_jited_linfo()  argument
208 	if (!prog->aux->jited_linfo)  in bpf_prog_fill_jited_linfo()
212 	linfo_idx = prog->aux->linfo_idx;  in bpf_prog_fill_jited_linfo()
213 	linfo = &prog->aux->linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
215 	insn_end = insn_start + prog->len;  in bpf_prog_fill_jited_linfo()
217 	jited_linfo = &prog->aux->jited_linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
218 	jited_linfo[0] = prog->bpf_func;  in bpf_prog_fill_jited_linfo()
220 	nr_linfo = prog->aux->nr_linfo - linfo_idx;  in bpf_prog_fill_jited_linfo()
226 		jited_linfo[i] = prog->bpf_func +  in bpf_prog_fill_jited_linfo()
246 		fp->aux->prog = fp;  in bpf_prog_realloc()
382 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old,  in bpf_adj_branches()  argument
385 	u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0);  in bpf_adj_branches()
386 	struct bpf_insn *insn = prog->insnsi;  in bpf_adj_branches()
398 			insn = prog->insnsi + end_old;  in bpf_adj_branches()
429 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)  in bpf_adj_linfo()  argument
434 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo()
438 	linfo = prog->aux->linfo;  in bpf_adj_linfo()
449 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,  in bpf_patch_insn_single()  argument
459 		memcpy(prog->insnsi + off, patch, sizeof(*patch));  in bpf_patch_insn_single()
460 		return prog;  in bpf_patch_insn_single()
463 	insn_adj_cnt = prog->len + insn_delta;  in bpf_patch_insn_single()
471 	    (err = bpf_adj_branches(prog, off, off + 1, off + len, true)))  in bpf_patch_insn_single()
478 	prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),  in bpf_patch_insn_single()
510 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt)  in bpf_remove_insns()  argument
515 	memmove(prog->insnsi + off, prog->insnsi + off + cnt,  in bpf_remove_insns()
516 		sizeof(struct bpf_insn) * (prog->len - off - cnt));  in bpf_remove_insns()
517 	prog->len -= cnt;  in bpf_remove_insns()
519 	return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false));  in bpf_remove_insns()
545 bpf_prog_ksym_set_addr(struct bpf_prog *prog)  in bpf_prog_ksym_set_addr()  argument
547 	WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog));  in bpf_prog_ksym_set_addr()
549 	prog->aux->ksym.start = (unsigned long) prog->bpf_func;  in bpf_prog_ksym_set_addr()
550 	prog->aux->ksym.end   = prog->aux->ksym.start + prog->jited_len;  in bpf_prog_ksym_set_addr()
554 bpf_prog_ksym_set_name(struct bpf_prog *prog)  in bpf_prog_ksym_set_name()  argument
556 	char *sym = prog->aux->ksym.name;  in bpf_prog_ksym_set_name()
562 		     sizeof(prog->tag) * 2 +  in bpf_prog_ksym_set_name()
570 		     sizeof(prog->aux->name) > KSYM_NAME_LEN);  in bpf_prog_ksym_set_name()
573 	sym  = bin2hex(sym, prog->tag, sizeof(prog->tag));  in bpf_prog_ksym_set_name()
576 	if (prog->aux->func_info_cnt) {  in bpf_prog_ksym_set_name()
577 		type = btf_type_by_id(prog->aux->btf,  in bpf_prog_ksym_set_name()
578 				      prog->aux->func_info[prog->aux->func_idx].type_id);  in bpf_prog_ksym_set_name()
579 		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);  in bpf_prog_ksym_set_name()
584 	if (prog->aux->name[0])  in bpf_prog_ksym_set_name()
585 		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);  in bpf_prog_ksym_set_name()
663 	fp->aux->ksym.prog = true;  in bpf_prog_kallsyms_add()
724 	return ksym && ksym->prog ?  in bpf_prog_ksym_find()
725 	       container_of(ksym, struct bpf_prog_aux, ksym)->prog :  in bpf_prog_ksym_find()
732 	struct bpf_prog *prog;  in search_bpf_extables()  local
735 	prog = bpf_prog_ksym_find(addr);  in search_bpf_extables()
736 	if (!prog)  in search_bpf_extables()
738 	if (!prog->aux->num_exentries)  in search_bpf_extables()
741 	e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);  in search_bpf_extables()
775 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,  in bpf_jit_add_poke_descriptor()  argument
778 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in bpf_jit_add_poke_descriptor()
780 	u32 slot = prog->aux->size_poke_tab;  in bpf_jit_add_poke_descriptor()
803 	prog->aux->size_poke_tab = size;  in bpf_jit_add_poke_descriptor()
804 	prog->aux->poke_tab = tab;  in bpf_jit_add_poke_descriptor()
1106 int bpf_jit_binary_pack_finalize(struct bpf_prog *prog,  in bpf_jit_binary_pack_finalize()  argument
1179 int bpf_jit_get_func_addr(const struct bpf_prog *prog,  in bpf_jit_get_func_addr()  argument
1195 		else if (prog->aux->func &&  in bpf_jit_get_func_addr()
1196 			 off >= 0 && off < prog->aux->func_cnt)  in bpf_jit_get_func_addr()
1197 			addr = (u8 *)prog->aux->func[off]->bpf_func;  in bpf_jit_get_func_addr()
1387 	fp->aux->prog = fp;  in bpf_jit_prog_release_other()
1391 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)  in bpf_jit_blind_constants()  argument
1399 	if (!prog->blinding_requested || prog->blinded)  in bpf_jit_blind_constants()
1400 		return prog;  in bpf_jit_blind_constants()
1402 	clone = bpf_prog_clone_create(prog, GFP_USER);  in bpf_jit_blind_constants()
1439 			bpf_jit_prog_release_other(prog, clone);  in bpf_jit_blind_constants()
1832 		struct bpf_prog *prog;  in ___bpf_prog_run()  local
1843 		prog = READ_ONCE(array->ptrs[index]);  in ___bpf_prog_run()
1844 		if (!prog)  in ___bpf_prog_run()
1852 		insn = prog->insnsi;  in ___bpf_prog_run()
2221 	struct bpf_prog prog;  member
2223 	.prog = {
2271 	for (item = array->items; item->prog; item++)  in bpf_prog_array_length()
2272 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_length()
2281 	for (item = array->items; item->prog; item++)  in bpf_prog_array_is_empty()
2282 		if (item->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_is_empty()
2294 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_copy_core()
2295 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy_core()
2297 		prog_ids[i] = item->prog->aux->id;  in bpf_prog_array_copy_core()
2304 	return !!(item->prog);  in bpf_prog_array_copy_core()
2338 	for (item = array->items; item->prog; item++)  in bpf_prog_array_delete_safe()
2339 		if (item->prog == old_prog) {  in bpf_prog_array_delete_safe()
2340 			WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe()
2362 	return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog);  in bpf_prog_array_delete_safe_at()
2381 			     struct bpf_prog *prog)  in bpf_prog_array_update_at()  argument
2388 	for (item = array->items; item->prog; item++) {  in bpf_prog_array_update_at()
2389 		if (item->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_update_at()
2392 			WRITE_ONCE(item->prog, prog);  in bpf_prog_array_update_at()
2416 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
2417 			if (existing->prog == exclude_prog) {  in bpf_prog_array_copy()
2421 			if (existing->prog != &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
2423 			if (existing->prog == include_prog)  in bpf_prog_array_copy()
2451 		for (; existing->prog; existing++) {  in bpf_prog_array_copy()
2452 			if (existing->prog == exclude_prog ||  in bpf_prog_array_copy()
2453 			    existing->prog == &dummy_bpf_prog.prog)  in bpf_prog_array_copy()
2456 			new->prog = existing->prog;  in bpf_prog_array_copy()
2462 		new->prog = include_prog;  in bpf_prog_array_copy()
2466 	new->prog = NULL;  in bpf_prog_array_copy()
2549 		bpf_prog_offload_destroy(aux->prog);  in bpf_prog_free_deferred()
2551 	if (aux->prog->has_callchain_buf)  in bpf_prog_free_deferred()
2566 		bpf_prog_unlock_free(aux->prog);  in bpf_prog_free_deferred()
2568 		bpf_jit_free(aux->prog);  in bpf_prog_free_deferred()
2678 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
2680 	return prog;  in bpf_int_jit_compile()
2686 void __weak bpf_jit_compile(struct bpf_prog *prog)  in bpf_jit_compile()  argument