1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3  */
4 #include <linux/bpf.h>
5 #include <linux/btf.h>
6 #include <linux/bpf-cgroup.h>
7 #include <linux/rcupdate.h>
8 #include <linux/random.h>
9 #include <linux/smp.h>
10 #include <linux/topology.h>
11 #include <linux/ktime.h>
12 #include <linux/sched.h>
13 #include <linux/uidgid.h>
14 #include <linux/filter.h>
15 #include <linux/ctype.h>
16 #include <linux/jiffies.h>
17 #include <linux/pid_namespace.h>
18 #include <linux/poison.h>
19 #include <linux/proc_ns.h>
20 #include <linux/security.h>
21 #include <linux/btf_ids.h>
22 
23 #include "../../lib/kstrtox.h"
24 
25 /* If kernel subsystem is allowing eBPF programs to call this function,
26  * inside its own verifier_ops->get_func_proto() callback it should return
27  * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments
28  *
29  * Different map implementations will rely on rcu in map methods
30  * lookup/update/delete, therefore eBPF programs must run under rcu lock
31  * if program is allowed to access maps, so check rcu_read_lock_held in
32  * all three functions.
33  */
BPF_CALL_2(bpf_map_lookup_elem,struct bpf_map *,map,void *,key)34 BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
35 {
36 	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
37 	return (unsigned long) map->ops->map_lookup_elem(map, key);
38 }
39 
40 const struct bpf_func_proto bpf_map_lookup_elem_proto = {
41 	.func		= bpf_map_lookup_elem,
42 	.gpl_only	= false,
43 	.pkt_access	= true,
44 	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
45 	.arg1_type	= ARG_CONST_MAP_PTR,
46 	.arg2_type	= ARG_PTR_TO_MAP_KEY,
47 };
48 
BPF_CALL_4(bpf_map_update_elem,struct bpf_map *,map,void *,key,void *,value,u64,flags)49 BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
50 	   void *, value, u64, flags)
51 {
52 	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
53 	return map->ops->map_update_elem(map, key, value, flags);
54 }
55 
56 const struct bpf_func_proto bpf_map_update_elem_proto = {
57 	.func		= bpf_map_update_elem,
58 	.gpl_only	= false,
59 	.pkt_access	= true,
60 	.ret_type	= RET_INTEGER,
61 	.arg1_type	= ARG_CONST_MAP_PTR,
62 	.arg2_type	= ARG_PTR_TO_MAP_KEY,
63 	.arg3_type	= ARG_PTR_TO_MAP_VALUE,
64 	.arg4_type	= ARG_ANYTHING,
65 };
66 
BPF_CALL_2(bpf_map_delete_elem,struct bpf_map *,map,void *,key)67 BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
68 {
69 	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
70 	return map->ops->map_delete_elem(map, key);
71 }
72 
73 const struct bpf_func_proto bpf_map_delete_elem_proto = {
74 	.func		= bpf_map_delete_elem,
75 	.gpl_only	= false,
76 	.pkt_access	= true,
77 	.ret_type	= RET_INTEGER,
78 	.arg1_type	= ARG_CONST_MAP_PTR,
79 	.arg2_type	= ARG_PTR_TO_MAP_KEY,
80 };
81 
BPF_CALL_3(bpf_map_push_elem,struct bpf_map *,map,void *,value,u64,flags)82 BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags)
83 {
84 	return map->ops->map_push_elem(map, value, flags);
85 }
86 
87 const struct bpf_func_proto bpf_map_push_elem_proto = {
88 	.func		= bpf_map_push_elem,
89 	.gpl_only	= false,
90 	.pkt_access	= true,
91 	.ret_type	= RET_INTEGER,
92 	.arg1_type	= ARG_CONST_MAP_PTR,
93 	.arg2_type	= ARG_PTR_TO_MAP_VALUE,
94 	.arg3_type	= ARG_ANYTHING,
95 };
96 
BPF_CALL_2(bpf_map_pop_elem,struct bpf_map *,map,void *,value)97 BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
98 {
99 	return map->ops->map_pop_elem(map, value);
100 }
101 
102 const struct bpf_func_proto bpf_map_pop_elem_proto = {
103 	.func		= bpf_map_pop_elem,
104 	.gpl_only	= false,
105 	.ret_type	= RET_INTEGER,
106 	.arg1_type	= ARG_CONST_MAP_PTR,
107 	.arg2_type	= ARG_PTR_TO_MAP_VALUE | MEM_UNINIT,
108 };
109 
BPF_CALL_2(bpf_map_peek_elem,struct bpf_map *,map,void *,value)110 BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
111 {
112 	return map->ops->map_peek_elem(map, value);
113 }
114 
115 const struct bpf_func_proto bpf_map_peek_elem_proto = {
116 	.func		= bpf_map_peek_elem,
117 	.gpl_only	= false,
118 	.ret_type	= RET_INTEGER,
119 	.arg1_type	= ARG_CONST_MAP_PTR,
120 	.arg2_type	= ARG_PTR_TO_MAP_VALUE | MEM_UNINIT,
121 };
122 
BPF_CALL_3(bpf_map_lookup_percpu_elem,struct bpf_map *,map,void *,key,u32,cpu)123 BPF_CALL_3(bpf_map_lookup_percpu_elem, struct bpf_map *, map, void *, key, u32, cpu)
124 {
125 	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
126 	return (unsigned long) map->ops->map_lookup_percpu_elem(map, key, cpu);
127 }
128 
129 const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto = {
130 	.func		= bpf_map_lookup_percpu_elem,
131 	.gpl_only	= false,
132 	.pkt_access	= true,
133 	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
134 	.arg1_type	= ARG_CONST_MAP_PTR,
135 	.arg2_type	= ARG_PTR_TO_MAP_KEY,
136 	.arg3_type	= ARG_ANYTHING,
137 };
138 
139 const struct bpf_func_proto bpf_get_prandom_u32_proto = {
140 	.func		= bpf_user_rnd_u32,
141 	.gpl_only	= false,
142 	.ret_type	= RET_INTEGER,
143 };
144 
BPF_CALL_0(bpf_get_smp_processor_id)145 BPF_CALL_0(bpf_get_smp_processor_id)
146 {
147 	return smp_processor_id();
148 }
149 
150 const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
151 	.func		= bpf_get_smp_processor_id,
152 	.gpl_only	= false,
153 	.ret_type	= RET_INTEGER,
154 };
155 
BPF_CALL_0(bpf_get_numa_node_id)156 BPF_CALL_0(bpf_get_numa_node_id)
157 {
158 	return numa_node_id();
159 }
160 
161 const struct bpf_func_proto bpf_get_numa_node_id_proto = {
162 	.func		= bpf_get_numa_node_id,
163 	.gpl_only	= false,
164 	.ret_type	= RET_INTEGER,
165 };
166 
BPF_CALL_0(bpf_ktime_get_ns)167 BPF_CALL_0(bpf_ktime_get_ns)
168 {
169 	/* NMI safe access to clock monotonic */
170 	return ktime_get_mono_fast_ns();
171 }
172 
173 const struct bpf_func_proto bpf_ktime_get_ns_proto = {
174 	.func		= bpf_ktime_get_ns,
175 	.gpl_only	= false,
176 	.ret_type	= RET_INTEGER,
177 };
178 
BPF_CALL_0(bpf_ktime_get_boot_ns)179 BPF_CALL_0(bpf_ktime_get_boot_ns)
180 {
181 	/* NMI safe access to clock boottime */
182 	return ktime_get_boot_fast_ns();
183 }
184 
185 const struct bpf_func_proto bpf_ktime_get_boot_ns_proto = {
186 	.func		= bpf_ktime_get_boot_ns,
187 	.gpl_only	= false,
188 	.ret_type	= RET_INTEGER,
189 };
190 
BPF_CALL_0(bpf_ktime_get_coarse_ns)191 BPF_CALL_0(bpf_ktime_get_coarse_ns)
192 {
193 	return ktime_get_coarse_ns();
194 }
195 
196 const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto = {
197 	.func		= bpf_ktime_get_coarse_ns,
198 	.gpl_only	= false,
199 	.ret_type	= RET_INTEGER,
200 };
201 
BPF_CALL_0(bpf_ktime_get_tai_ns)202 BPF_CALL_0(bpf_ktime_get_tai_ns)
203 {
204 	/* NMI safe access to clock tai */
205 	return ktime_get_tai_fast_ns();
206 }
207 
208 const struct bpf_func_proto bpf_ktime_get_tai_ns_proto = {
209 	.func		= bpf_ktime_get_tai_ns,
210 	.gpl_only	= false,
211 	.ret_type	= RET_INTEGER,
212 };
213 
BPF_CALL_0(bpf_get_current_pid_tgid)214 BPF_CALL_0(bpf_get_current_pid_tgid)
215 {
216 	struct task_struct *task = current;
217 
218 	if (unlikely(!task))
219 		return -EINVAL;
220 
221 	return (u64) task->tgid << 32 | task->pid;
222 }
223 
224 const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {
225 	.func		= bpf_get_current_pid_tgid,
226 	.gpl_only	= false,
227 	.ret_type	= RET_INTEGER,
228 };
229 
BPF_CALL_0(bpf_get_current_uid_gid)230 BPF_CALL_0(bpf_get_current_uid_gid)
231 {
232 	struct task_struct *task = current;
233 	kuid_t uid;
234 	kgid_t gid;
235 
236 	if (unlikely(!task))
237 		return -EINVAL;
238 
239 	current_uid_gid(&uid, &gid);
240 	return (u64) from_kgid(&init_user_ns, gid) << 32 |
241 		     from_kuid(&init_user_ns, uid);
242 }
243 
244 const struct bpf_func_proto bpf_get_current_uid_gid_proto = {
245 	.func		= bpf_get_current_uid_gid,
246 	.gpl_only	= false,
247 	.ret_type	= RET_INTEGER,
248 };
249 
BPF_CALL_2(bpf_get_current_comm,char *,buf,u32,size)250 BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)
251 {
252 	struct task_struct *task = current;
253 
254 	if (unlikely(!task))
255 		goto err_clear;
256 
257 	/* Verifier guarantees that size > 0 */
258 	strscpy(buf, task->comm, size);
259 	return 0;
260 err_clear:
261 	memset(buf, 0, size);
262 	return -EINVAL;
263 }
264 
265 const struct bpf_func_proto bpf_get_current_comm_proto = {
266 	.func		= bpf_get_current_comm,
267 	.gpl_only	= false,
268 	.ret_type	= RET_INTEGER,
269 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
270 	.arg2_type	= ARG_CONST_SIZE,
271 };
272 
273 #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK)
274 
__bpf_spin_lock(struct bpf_spin_lock * lock)275 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
276 {
277 	arch_spinlock_t *l = (void *)lock;
278 	union {
279 		__u32 val;
280 		arch_spinlock_t lock;
281 	} u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED };
282 
283 	compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");
284 	BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));
285 	BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));
286 	arch_spin_lock(l);
287 }
288 
__bpf_spin_unlock(struct bpf_spin_lock * lock)289 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
290 {
291 	arch_spinlock_t *l = (void *)lock;
292 
293 	arch_spin_unlock(l);
294 }
295 
296 #else
297 
__bpf_spin_lock(struct bpf_spin_lock * lock)298 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
299 {
300 	atomic_t *l = (void *)lock;
301 
302 	BUILD_BUG_ON(sizeof(*l) != sizeof(*lock));
303 	do {
304 		atomic_cond_read_relaxed(l, !VAL);
305 	} while (atomic_xchg(l, 1));
306 }
307 
__bpf_spin_unlock(struct bpf_spin_lock * lock)308 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
309 {
310 	atomic_t *l = (void *)lock;
311 
312 	atomic_set_release(l, 0);
313 }
314 
315 #endif
316 
317 static DEFINE_PER_CPU(unsigned long, irqsave_flags);
318 
__bpf_spin_lock_irqsave(struct bpf_spin_lock * lock)319 static inline void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock)
320 {
321 	unsigned long flags;
322 
323 	local_irq_save(flags);
324 	__bpf_spin_lock(lock);
325 	__this_cpu_write(irqsave_flags, flags);
326 }
327 
BPF_CALL_1(bpf_spin_lock,struct bpf_spin_lock *,lock)328 notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
329 {
330 	__bpf_spin_lock_irqsave(lock);
331 	return 0;
332 }
333 
334 const struct bpf_func_proto bpf_spin_lock_proto = {
335 	.func		= bpf_spin_lock,
336 	.gpl_only	= false,
337 	.ret_type	= RET_VOID,
338 	.arg1_type	= ARG_PTR_TO_SPIN_LOCK,
339 };
340 
__bpf_spin_unlock_irqrestore(struct bpf_spin_lock * lock)341 static inline void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock)
342 {
343 	unsigned long flags;
344 
345 	flags = __this_cpu_read(irqsave_flags);
346 	__bpf_spin_unlock(lock);
347 	local_irq_restore(flags);
348 }
349 
BPF_CALL_1(bpf_spin_unlock,struct bpf_spin_lock *,lock)350 notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
351 {
352 	__bpf_spin_unlock_irqrestore(lock);
353 	return 0;
354 }
355 
356 const struct bpf_func_proto bpf_spin_unlock_proto = {
357 	.func		= bpf_spin_unlock,
358 	.gpl_only	= false,
359 	.ret_type	= RET_VOID,
360 	.arg1_type	= ARG_PTR_TO_SPIN_LOCK,
361 };
362 
copy_map_value_locked(struct bpf_map * map,void * dst,void * src,bool lock_src)363 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
364 			   bool lock_src)
365 {
366 	struct bpf_spin_lock *lock;
367 
368 	if (lock_src)
369 		lock = src + map->spin_lock_off;
370 	else
371 		lock = dst + map->spin_lock_off;
372 	preempt_disable();
373 	__bpf_spin_lock_irqsave(lock);
374 	copy_map_value(map, dst, src);
375 	__bpf_spin_unlock_irqrestore(lock);
376 	preempt_enable();
377 }
378 
BPF_CALL_0(bpf_jiffies64)379 BPF_CALL_0(bpf_jiffies64)
380 {
381 	return get_jiffies_64();
382 }
383 
384 const struct bpf_func_proto bpf_jiffies64_proto = {
385 	.func		= bpf_jiffies64,
386 	.gpl_only	= false,
387 	.ret_type	= RET_INTEGER,
388 };
389 
390 #ifdef CONFIG_CGROUPS
BPF_CALL_0(bpf_get_current_cgroup_id)391 BPF_CALL_0(bpf_get_current_cgroup_id)
392 {
393 	struct cgroup *cgrp;
394 	u64 cgrp_id;
395 
396 	rcu_read_lock();
397 	cgrp = task_dfl_cgroup(current);
398 	cgrp_id = cgroup_id(cgrp);
399 	rcu_read_unlock();
400 
401 	return cgrp_id;
402 }
403 
404 const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
405 	.func		= bpf_get_current_cgroup_id,
406 	.gpl_only	= false,
407 	.ret_type	= RET_INTEGER,
408 };
409 
BPF_CALL_1(bpf_get_current_ancestor_cgroup_id,int,ancestor_level)410 BPF_CALL_1(bpf_get_current_ancestor_cgroup_id, int, ancestor_level)
411 {
412 	struct cgroup *cgrp;
413 	struct cgroup *ancestor;
414 	u64 cgrp_id;
415 
416 	rcu_read_lock();
417 	cgrp = task_dfl_cgroup(current);
418 	ancestor = cgroup_ancestor(cgrp, ancestor_level);
419 	cgrp_id = ancestor ? cgroup_id(ancestor) : 0;
420 	rcu_read_unlock();
421 
422 	return cgrp_id;
423 }
424 
425 const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = {
426 	.func		= bpf_get_current_ancestor_cgroup_id,
427 	.gpl_only	= false,
428 	.ret_type	= RET_INTEGER,
429 	.arg1_type	= ARG_ANYTHING,
430 };
431 #endif /* CONFIG_CGROUPS */
432 
433 #define BPF_STRTOX_BASE_MASK 0x1F
434 
__bpf_strtoull(const char * buf,size_t buf_len,u64 flags,unsigned long long * res,bool * is_negative)435 static int __bpf_strtoull(const char *buf, size_t buf_len, u64 flags,
436 			  unsigned long long *res, bool *is_negative)
437 {
438 	unsigned int base = flags & BPF_STRTOX_BASE_MASK;
439 	const char *cur_buf = buf;
440 	size_t cur_len = buf_len;
441 	unsigned int consumed;
442 	size_t val_len;
443 	char str[64];
444 
445 	if (!buf || !buf_len || !res || !is_negative)
446 		return -EINVAL;
447 
448 	if (base != 0 && base != 8 && base != 10 && base != 16)
449 		return -EINVAL;
450 
451 	if (flags & ~BPF_STRTOX_BASE_MASK)
452 		return -EINVAL;
453 
454 	while (cur_buf < buf + buf_len && isspace(*cur_buf))
455 		++cur_buf;
456 
457 	*is_negative = (cur_buf < buf + buf_len && *cur_buf == '-');
458 	if (*is_negative)
459 		++cur_buf;
460 
461 	consumed = cur_buf - buf;
462 	cur_len -= consumed;
463 	if (!cur_len)
464 		return -EINVAL;
465 
466 	cur_len = min(cur_len, sizeof(str) - 1);
467 	memcpy(str, cur_buf, cur_len);
468 	str[cur_len] = '\0';
469 	cur_buf = str;
470 
471 	cur_buf = _parse_integer_fixup_radix(cur_buf, &base);
472 	val_len = _parse_integer(cur_buf, base, res);
473 
474 	if (val_len & KSTRTOX_OVERFLOW)
475 		return -ERANGE;
476 
477 	if (val_len == 0)
478 		return -EINVAL;
479 
480 	cur_buf += val_len;
481 	consumed += cur_buf - str;
482 
483 	return consumed;
484 }
485 
__bpf_strtoll(const char * buf,size_t buf_len,u64 flags,long long * res)486 static int __bpf_strtoll(const char *buf, size_t buf_len, u64 flags,
487 			 long long *res)
488 {
489 	unsigned long long _res;
490 	bool is_negative;
491 	int err;
492 
493 	err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);
494 	if (err < 0)
495 		return err;
496 	if (is_negative) {
497 		if ((long long)-_res > 0)
498 			return -ERANGE;
499 		*res = -_res;
500 	} else {
501 		if ((long long)_res < 0)
502 			return -ERANGE;
503 		*res = _res;
504 	}
505 	return err;
506 }
507 
BPF_CALL_4(bpf_strtol,const char *,buf,size_t,buf_len,u64,flags,long *,res)508 BPF_CALL_4(bpf_strtol, const char *, buf, size_t, buf_len, u64, flags,
509 	   long *, res)
510 {
511 	long long _res;
512 	int err;
513 
514 	err = __bpf_strtoll(buf, buf_len, flags, &_res);
515 	if (err < 0)
516 		return err;
517 	if (_res != (long)_res)
518 		return -ERANGE;
519 	*res = _res;
520 	return err;
521 }
522 
523 const struct bpf_func_proto bpf_strtol_proto = {
524 	.func		= bpf_strtol,
525 	.gpl_only	= false,
526 	.ret_type	= RET_INTEGER,
527 	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
528 	.arg2_type	= ARG_CONST_SIZE,
529 	.arg3_type	= ARG_ANYTHING,
530 	.arg4_type	= ARG_PTR_TO_LONG,
531 };
532 
BPF_CALL_4(bpf_strtoul,const char *,buf,size_t,buf_len,u64,flags,unsigned long *,res)533 BPF_CALL_4(bpf_strtoul, const char *, buf, size_t, buf_len, u64, flags,
534 	   unsigned long *, res)
535 {
536 	unsigned long long _res;
537 	bool is_negative;
538 	int err;
539 
540 	err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);
541 	if (err < 0)
542 		return err;
543 	if (is_negative)
544 		return -EINVAL;
545 	if (_res != (unsigned long)_res)
546 		return -ERANGE;
547 	*res = _res;
548 	return err;
549 }
550 
551 const struct bpf_func_proto bpf_strtoul_proto = {
552 	.func		= bpf_strtoul,
553 	.gpl_only	= false,
554 	.ret_type	= RET_INTEGER,
555 	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
556 	.arg2_type	= ARG_CONST_SIZE,
557 	.arg3_type	= ARG_ANYTHING,
558 	.arg4_type	= ARG_PTR_TO_LONG,
559 };
560 
BPF_CALL_3(bpf_strncmp,const char *,s1,u32,s1_sz,const char *,s2)561 BPF_CALL_3(bpf_strncmp, const char *, s1, u32, s1_sz, const char *, s2)
562 {
563 	return strncmp(s1, s2, s1_sz);
564 }
565 
566 static const struct bpf_func_proto bpf_strncmp_proto = {
567 	.func		= bpf_strncmp,
568 	.gpl_only	= false,
569 	.ret_type	= RET_INTEGER,
570 	.arg1_type	= ARG_PTR_TO_MEM,
571 	.arg2_type	= ARG_CONST_SIZE,
572 	.arg3_type	= ARG_PTR_TO_CONST_STR,
573 };
574 
BPF_CALL_4(bpf_get_ns_current_pid_tgid,u64,dev,u64,ino,struct bpf_pidns_info *,nsdata,u32,size)575 BPF_CALL_4(bpf_get_ns_current_pid_tgid, u64, dev, u64, ino,
576 	   struct bpf_pidns_info *, nsdata, u32, size)
577 {
578 	struct task_struct *task = current;
579 	struct pid_namespace *pidns;
580 	int err = -EINVAL;
581 
582 	if (unlikely(size != sizeof(struct bpf_pidns_info)))
583 		goto clear;
584 
585 	if (unlikely((u64)(dev_t)dev != dev))
586 		goto clear;
587 
588 	if (unlikely(!task))
589 		goto clear;
590 
591 	pidns = task_active_pid_ns(task);
592 	if (unlikely(!pidns)) {
593 		err = -ENOENT;
594 		goto clear;
595 	}
596 
597 	if (!ns_match(&pidns->ns, (dev_t)dev, ino))
598 		goto clear;
599 
600 	nsdata->pid = task_pid_nr_ns(task, pidns);
601 	nsdata->tgid = task_tgid_nr_ns(task, pidns);
602 	return 0;
603 clear:
604 	memset((void *)nsdata, 0, (size_t) size);
605 	return err;
606 }
607 
608 const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto = {
609 	.func		= bpf_get_ns_current_pid_tgid,
610 	.gpl_only	= false,
611 	.ret_type	= RET_INTEGER,
612 	.arg1_type	= ARG_ANYTHING,
613 	.arg2_type	= ARG_ANYTHING,
614 	.arg3_type      = ARG_PTR_TO_UNINIT_MEM,
615 	.arg4_type      = ARG_CONST_SIZE,
616 };
617 
618 static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = {
619 	.func		= bpf_get_raw_cpu_id,
620 	.gpl_only	= false,
621 	.ret_type	= RET_INTEGER,
622 };
623 
BPF_CALL_5(bpf_event_output_data,void *,ctx,struct bpf_map *,map,u64,flags,void *,data,u64,size)624 BPF_CALL_5(bpf_event_output_data, void *, ctx, struct bpf_map *, map,
625 	   u64, flags, void *, data, u64, size)
626 {
627 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
628 		return -EINVAL;
629 
630 	return bpf_event_output(map, flags, data, size, NULL, 0, NULL);
631 }
632 
633 const struct bpf_func_proto bpf_event_output_data_proto =  {
634 	.func		= bpf_event_output_data,
635 	.gpl_only       = true,
636 	.ret_type       = RET_INTEGER,
637 	.arg1_type      = ARG_PTR_TO_CTX,
638 	.arg2_type      = ARG_CONST_MAP_PTR,
639 	.arg3_type      = ARG_ANYTHING,
640 	.arg4_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
641 	.arg5_type      = ARG_CONST_SIZE_OR_ZERO,
642 };
643 
BPF_CALL_3(bpf_copy_from_user,void *,dst,u32,size,const void __user *,user_ptr)644 BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size,
645 	   const void __user *, user_ptr)
646 {
647 	int ret = copy_from_user(dst, user_ptr, size);
648 
649 	if (unlikely(ret)) {
650 		memset(dst, 0, size);
651 		ret = -EFAULT;
652 	}
653 
654 	return ret;
655 }
656 
657 const struct bpf_func_proto bpf_copy_from_user_proto = {
658 	.func		= bpf_copy_from_user,
659 	.gpl_only	= false,
660 	.ret_type	= RET_INTEGER,
661 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
662 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
663 	.arg3_type	= ARG_ANYTHING,
664 };
665 
BPF_CALL_5(bpf_copy_from_user_task,void *,dst,u32,size,const void __user *,user_ptr,struct task_struct *,tsk,u64,flags)666 BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size,
667 	   const void __user *, user_ptr, struct task_struct *, tsk, u64, flags)
668 {
669 	int ret;
670 
671 	/* flags is not used yet */
672 	if (unlikely(flags))
673 		return -EINVAL;
674 
675 	if (unlikely(!size))
676 		return 0;
677 
678 	ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
679 	if (ret == size)
680 		return 0;
681 
682 	memset(dst, 0, size);
683 	/* Return -EFAULT for partial read */
684 	return ret < 0 ? ret : -EFAULT;
685 }
686 
687 const struct bpf_func_proto bpf_copy_from_user_task_proto = {
688 	.func		= bpf_copy_from_user_task,
689 	.gpl_only	= true,
690 	.ret_type	= RET_INTEGER,
691 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
692 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
693 	.arg3_type	= ARG_ANYTHING,
694 	.arg4_type	= ARG_PTR_TO_BTF_ID,
695 	.arg4_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
696 	.arg5_type	= ARG_ANYTHING
697 };
698 
BPF_CALL_2(bpf_per_cpu_ptr,const void *,ptr,u32,cpu)699 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
700 {
701 	if (cpu >= nr_cpu_ids)
702 		return (unsigned long)NULL;
703 
704 	return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
705 }
706 
707 const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
708 	.func		= bpf_per_cpu_ptr,
709 	.gpl_only	= false,
710 	.ret_type	= RET_PTR_TO_MEM_OR_BTF_ID | PTR_MAYBE_NULL | MEM_RDONLY,
711 	.arg1_type	= ARG_PTR_TO_PERCPU_BTF_ID,
712 	.arg2_type	= ARG_ANYTHING,
713 };
714 
BPF_CALL_1(bpf_this_cpu_ptr,const void *,percpu_ptr)715 BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
716 {
717 	return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);
718 }
719 
720 const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
721 	.func		= bpf_this_cpu_ptr,
722 	.gpl_only	= false,
723 	.ret_type	= RET_PTR_TO_MEM_OR_BTF_ID | MEM_RDONLY,
724 	.arg1_type	= ARG_PTR_TO_PERCPU_BTF_ID,
725 };
726 
bpf_trace_copy_string(char * buf,void * unsafe_ptr,char fmt_ptype,size_t bufsz)727 static int bpf_trace_copy_string(char *buf, void *unsafe_ptr, char fmt_ptype,
728 		size_t bufsz)
729 {
730 	void __user *user_ptr = (__force void __user *)unsafe_ptr;
731 
732 	buf[0] = 0;
733 
734 	switch (fmt_ptype) {
735 	case 's':
736 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
737 		if ((unsigned long)unsafe_ptr < TASK_SIZE)
738 			return strncpy_from_user_nofault(buf, user_ptr, bufsz);
739 		fallthrough;
740 #endif
741 	case 'k':
742 		return strncpy_from_kernel_nofault(buf, unsafe_ptr, bufsz);
743 	case 'u':
744 		return strncpy_from_user_nofault(buf, user_ptr, bufsz);
745 	}
746 
747 	return -EINVAL;
748 }
749 
750 /* Per-cpu temp buffers used by printf-like helpers to store the bprintf binary
751  * arguments representation.
752  */
753 #define MAX_BPRINTF_BUF_LEN	512
754 
755 /* Support executing three nested bprintf helper calls on a given CPU */
756 #define MAX_BPRINTF_NEST_LEVEL	3
757 struct bpf_bprintf_buffers {
758 	char tmp_bufs[MAX_BPRINTF_NEST_LEVEL][MAX_BPRINTF_BUF_LEN];
759 };
760 static DEFINE_PER_CPU(struct bpf_bprintf_buffers, bpf_bprintf_bufs);
761 static DEFINE_PER_CPU(int, bpf_bprintf_nest_level);
762 
try_get_fmt_tmp_buf(char ** tmp_buf)763 static int try_get_fmt_tmp_buf(char **tmp_buf)
764 {
765 	struct bpf_bprintf_buffers *bufs;
766 	int nest_level;
767 
768 	preempt_disable();
769 	nest_level = this_cpu_inc_return(bpf_bprintf_nest_level);
770 	if (WARN_ON_ONCE(nest_level > MAX_BPRINTF_NEST_LEVEL)) {
771 		this_cpu_dec(bpf_bprintf_nest_level);
772 		preempt_enable();
773 		return -EBUSY;
774 	}
775 	bufs = this_cpu_ptr(&bpf_bprintf_bufs);
776 	*tmp_buf = bufs->tmp_bufs[nest_level - 1];
777 
778 	return 0;
779 }
780 
bpf_bprintf_cleanup(void)781 void bpf_bprintf_cleanup(void)
782 {
783 	if (this_cpu_read(bpf_bprintf_nest_level)) {
784 		this_cpu_dec(bpf_bprintf_nest_level);
785 		preempt_enable();
786 	}
787 }
788 
789 /*
790  * bpf_bprintf_prepare - Generic pass on format strings for bprintf-like helpers
791  *
792  * Returns a negative value if fmt is an invalid format string or 0 otherwise.
793  *
794  * This can be used in two ways:
795  * - Format string verification only: when bin_args is NULL
796  * - Arguments preparation: in addition to the above verification, it writes in
797  *   bin_args a binary representation of arguments usable by bstr_printf where
798  *   pointers from BPF have been sanitized.
799  *
800  * In argument preparation mode, if 0 is returned, safe temporary buffers are
801  * allocated and bpf_bprintf_cleanup should be called to free them after use.
802  */
bpf_bprintf_prepare(char * fmt,u32 fmt_size,const u64 * raw_args,u32 ** bin_args,u32 num_args)803 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
804 			u32 **bin_args, u32 num_args)
805 {
806 	char *unsafe_ptr = NULL, *tmp_buf = NULL, *tmp_buf_end, *fmt_end;
807 	size_t sizeof_cur_arg, sizeof_cur_ip;
808 	int err, i, num_spec = 0;
809 	u64 cur_arg;
810 	char fmt_ptype, cur_ip[16], ip_spec[] = "%pXX";
811 
812 	fmt_end = strnchr(fmt, fmt_size, 0);
813 	if (!fmt_end)
814 		return -EINVAL;
815 	fmt_size = fmt_end - fmt;
816 
817 	if (bin_args) {
818 		if (num_args && try_get_fmt_tmp_buf(&tmp_buf))
819 			return -EBUSY;
820 
821 		tmp_buf_end = tmp_buf + MAX_BPRINTF_BUF_LEN;
822 		*bin_args = (u32 *)tmp_buf;
823 	}
824 
825 	for (i = 0; i < fmt_size; i++) {
826 		if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) {
827 			err = -EINVAL;
828 			goto out;
829 		}
830 
831 		if (fmt[i] != '%')
832 			continue;
833 
834 		if (fmt[i + 1] == '%') {
835 			i++;
836 			continue;
837 		}
838 
839 		if (num_spec >= num_args) {
840 			err = -EINVAL;
841 			goto out;
842 		}
843 
844 		/* The string is zero-terminated so if fmt[i] != 0, we can
845 		 * always access fmt[i + 1], in the worst case it will be a 0
846 		 */
847 		i++;
848 
849 		/* skip optional "[0 +-][num]" width formatting field */
850 		while (fmt[i] == '0' || fmt[i] == '+'  || fmt[i] == '-' ||
851 		       fmt[i] == ' ')
852 			i++;
853 		if (fmt[i] >= '1' && fmt[i] <= '9') {
854 			i++;
855 			while (fmt[i] >= '0' && fmt[i] <= '9')
856 				i++;
857 		}
858 
859 		if (fmt[i] == 'p') {
860 			sizeof_cur_arg = sizeof(long);
861 
862 			if ((fmt[i + 1] == 'k' || fmt[i + 1] == 'u') &&
863 			    fmt[i + 2] == 's') {
864 				fmt_ptype = fmt[i + 1];
865 				i += 2;
866 				goto fmt_str;
867 			}
868 
869 			if (fmt[i + 1] == 0 || isspace(fmt[i + 1]) ||
870 			    ispunct(fmt[i + 1]) || fmt[i + 1] == 'K' ||
871 			    fmt[i + 1] == 'x' || fmt[i + 1] == 's' ||
872 			    fmt[i + 1] == 'S') {
873 				/* just kernel pointers */
874 				if (tmp_buf)
875 					cur_arg = raw_args[num_spec];
876 				i++;
877 				goto nocopy_fmt;
878 			}
879 
880 			if (fmt[i + 1] == 'B') {
881 				if (tmp_buf)  {
882 					err = snprintf(tmp_buf,
883 						       (tmp_buf_end - tmp_buf),
884 						       "%pB",
885 						       (void *)(long)raw_args[num_spec]);
886 					tmp_buf += (err + 1);
887 				}
888 
889 				i++;
890 				num_spec++;
891 				continue;
892 			}
893 
894 			/* only support "%pI4", "%pi4", "%pI6" and "%pi6". */
895 			if ((fmt[i + 1] != 'i' && fmt[i + 1] != 'I') ||
896 			    (fmt[i + 2] != '4' && fmt[i + 2] != '6')) {
897 				err = -EINVAL;
898 				goto out;
899 			}
900 
901 			i += 2;
902 			if (!tmp_buf)
903 				goto nocopy_fmt;
904 
905 			sizeof_cur_ip = (fmt[i] == '4') ? 4 : 16;
906 			if (tmp_buf_end - tmp_buf < sizeof_cur_ip) {
907 				err = -ENOSPC;
908 				goto out;
909 			}
910 
911 			unsafe_ptr = (char *)(long)raw_args[num_spec];
912 			err = copy_from_kernel_nofault(cur_ip, unsafe_ptr,
913 						       sizeof_cur_ip);
914 			if (err < 0)
915 				memset(cur_ip, 0, sizeof_cur_ip);
916 
917 			/* hack: bstr_printf expects IP addresses to be
918 			 * pre-formatted as strings, ironically, the easiest way
919 			 * to do that is to call snprintf.
920 			 */
921 			ip_spec[2] = fmt[i - 1];
922 			ip_spec[3] = fmt[i];
923 			err = snprintf(tmp_buf, tmp_buf_end - tmp_buf,
924 				       ip_spec, &cur_ip);
925 
926 			tmp_buf += err + 1;
927 			num_spec++;
928 
929 			continue;
930 		} else if (fmt[i] == 's') {
931 			fmt_ptype = fmt[i];
932 fmt_str:
933 			if (fmt[i + 1] != 0 &&
934 			    !isspace(fmt[i + 1]) &&
935 			    !ispunct(fmt[i + 1])) {
936 				err = -EINVAL;
937 				goto out;
938 			}
939 
940 			if (!tmp_buf)
941 				goto nocopy_fmt;
942 
943 			if (tmp_buf_end == tmp_buf) {
944 				err = -ENOSPC;
945 				goto out;
946 			}
947 
948 			unsafe_ptr = (char *)(long)raw_args[num_spec];
949 			err = bpf_trace_copy_string(tmp_buf, unsafe_ptr,
950 						    fmt_ptype,
951 						    tmp_buf_end - tmp_buf);
952 			if (err < 0) {
953 				tmp_buf[0] = '\0';
954 				err = 1;
955 			}
956 
957 			tmp_buf += err;
958 			num_spec++;
959 
960 			continue;
961 		} else if (fmt[i] == 'c') {
962 			if (!tmp_buf)
963 				goto nocopy_fmt;
964 
965 			if (tmp_buf_end == tmp_buf) {
966 				err = -ENOSPC;
967 				goto out;
968 			}
969 
970 			*tmp_buf = raw_args[num_spec];
971 			tmp_buf++;
972 			num_spec++;
973 
974 			continue;
975 		}
976 
977 		sizeof_cur_arg = sizeof(int);
978 
979 		if (fmt[i] == 'l') {
980 			sizeof_cur_arg = sizeof(long);
981 			i++;
982 		}
983 		if (fmt[i] == 'l') {
984 			sizeof_cur_arg = sizeof(long long);
985 			i++;
986 		}
987 
988 		if (fmt[i] != 'i' && fmt[i] != 'd' && fmt[i] != 'u' &&
989 		    fmt[i] != 'x' && fmt[i] != 'X') {
990 			err = -EINVAL;
991 			goto out;
992 		}
993 
994 		if (tmp_buf)
995 			cur_arg = raw_args[num_spec];
996 nocopy_fmt:
997 		if (tmp_buf) {
998 			tmp_buf = PTR_ALIGN(tmp_buf, sizeof(u32));
999 			if (tmp_buf_end - tmp_buf < sizeof_cur_arg) {
1000 				err = -ENOSPC;
1001 				goto out;
1002 			}
1003 
1004 			if (sizeof_cur_arg == 8) {
1005 				*(u32 *)tmp_buf = *(u32 *)&cur_arg;
1006 				*(u32 *)(tmp_buf + 4) = *((u32 *)&cur_arg + 1);
1007 			} else {
1008 				*(u32 *)tmp_buf = (u32)(long)cur_arg;
1009 			}
1010 			tmp_buf += sizeof_cur_arg;
1011 		}
1012 		num_spec++;
1013 	}
1014 
1015 	err = 0;
1016 out:
1017 	if (err)
1018 		bpf_bprintf_cleanup();
1019 	return err;
1020 }
1021 
BPF_CALL_5(bpf_snprintf,char *,str,u32,str_size,char *,fmt,const void *,data,u32,data_len)1022 BPF_CALL_5(bpf_snprintf, char *, str, u32, str_size, char *, fmt,
1023 	   const void *, data, u32, data_len)
1024 {
1025 	int err, num_args;
1026 	u32 *bin_args;
1027 
1028 	if (data_len % 8 || data_len > MAX_BPRINTF_VARARGS * 8 ||
1029 	    (data_len && !data))
1030 		return -EINVAL;
1031 	num_args = data_len / 8;
1032 
1033 	/* ARG_PTR_TO_CONST_STR guarantees that fmt is zero-terminated so we
1034 	 * can safely give an unbounded size.
1035 	 */
1036 	err = bpf_bprintf_prepare(fmt, UINT_MAX, data, &bin_args, num_args);
1037 	if (err < 0)
1038 		return err;
1039 
1040 	err = bstr_printf(str, str_size, fmt, bin_args);
1041 
1042 	bpf_bprintf_cleanup();
1043 
1044 	return err + 1;
1045 }
1046 
1047 const struct bpf_func_proto bpf_snprintf_proto = {
1048 	.func		= bpf_snprintf,
1049 	.gpl_only	= true,
1050 	.ret_type	= RET_INTEGER,
1051 	.arg1_type	= ARG_PTR_TO_MEM_OR_NULL,
1052 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
1053 	.arg3_type	= ARG_PTR_TO_CONST_STR,
1054 	.arg4_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
1055 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
1056 };
1057 
1058 /* BPF map elements can contain 'struct bpf_timer'.
1059  * Such map owns all of its BPF timers.
1060  * 'struct bpf_timer' is allocated as part of map element allocation
1061  * and it's zero initialized.
1062  * That space is used to keep 'struct bpf_timer_kern'.
1063  * bpf_timer_init() allocates 'struct bpf_hrtimer', inits hrtimer, and
1064  * remembers 'struct bpf_map *' pointer it's part of.
1065  * bpf_timer_set_callback() increments prog refcnt and assign bpf callback_fn.
1066  * bpf_timer_start() arms the timer.
1067  * If user space reference to a map goes to zero at this point
1068  * ops->map_release_uref callback is responsible for cancelling the timers,
1069  * freeing their memory, and decrementing prog's refcnts.
1070  * bpf_timer_cancel() cancels the timer and decrements prog's refcnt.
1071  * Inner maps can contain bpf timers as well. ops->map_release_uref is
1072  * freeing the timers when inner map is replaced or deleted by user space.
1073  */
1074 struct bpf_hrtimer {
1075 	struct hrtimer timer;
1076 	struct bpf_map *map;
1077 	struct bpf_prog *prog;
1078 	void __rcu *callback_fn;
1079 	void *value;
1080 };
1081 
1082 /* the actual struct hidden inside uapi struct bpf_timer */
1083 struct bpf_timer_kern {
1084 	struct bpf_hrtimer *timer;
1085 	/* bpf_spin_lock is used here instead of spinlock_t to make
1086 	 * sure that it always fits into space reserved by struct bpf_timer
1087 	 * regardless of LOCKDEP and spinlock debug flags.
1088 	 */
1089 	struct bpf_spin_lock lock;
1090 } __attribute__((aligned(8)));
1091 
1092 static DEFINE_PER_CPU(struct bpf_hrtimer *, hrtimer_running);
1093 
bpf_timer_cb(struct hrtimer * hrtimer)1094 static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer)
1095 {
1096 	struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer);
1097 	struct bpf_map *map = t->map;
1098 	void *value = t->value;
1099 	bpf_callback_t callback_fn;
1100 	void *key;
1101 	u32 idx;
1102 
1103 	BTF_TYPE_EMIT(struct bpf_timer);
1104 	callback_fn = rcu_dereference_check(t->callback_fn, rcu_read_lock_bh_held());
1105 	if (!callback_fn)
1106 		goto out;
1107 
1108 	/* bpf_timer_cb() runs in hrtimer_run_softirq. It doesn't migrate and
1109 	 * cannot be preempted by another bpf_timer_cb() on the same cpu.
1110 	 * Remember the timer this callback is servicing to prevent
1111 	 * deadlock if callback_fn() calls bpf_timer_cancel() or
1112 	 * bpf_map_delete_elem() on the same timer.
1113 	 */
1114 	this_cpu_write(hrtimer_running, t);
1115 	if (map->map_type == BPF_MAP_TYPE_ARRAY) {
1116 		struct bpf_array *array = container_of(map, struct bpf_array, map);
1117 
1118 		/* compute the key */
1119 		idx = ((char *)value - array->value) / array->elem_size;
1120 		key = &idx;
1121 	} else { /* hash or lru */
1122 		key = value - round_up(map->key_size, 8);
1123 	}
1124 
1125 	callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0);
1126 	/* The verifier checked that return value is zero. */
1127 
1128 	this_cpu_write(hrtimer_running, NULL);
1129 out:
1130 	return HRTIMER_NORESTART;
1131 }
1132 
BPF_CALL_3(bpf_timer_init,struct bpf_timer_kern *,timer,struct bpf_map *,map,u64,flags)1133 BPF_CALL_3(bpf_timer_init, struct bpf_timer_kern *, timer, struct bpf_map *, map,
1134 	   u64, flags)
1135 {
1136 	clockid_t clockid = flags & (MAX_CLOCKS - 1);
1137 	struct bpf_hrtimer *t;
1138 	int ret = 0;
1139 
1140 	BUILD_BUG_ON(MAX_CLOCKS != 16);
1141 	BUILD_BUG_ON(sizeof(struct bpf_timer_kern) > sizeof(struct bpf_timer));
1142 	BUILD_BUG_ON(__alignof__(struct bpf_timer_kern) != __alignof__(struct bpf_timer));
1143 
1144 	if (in_nmi())
1145 		return -EOPNOTSUPP;
1146 
1147 	if (flags >= MAX_CLOCKS ||
1148 	    /* similar to timerfd except _ALARM variants are not supported */
1149 	    (clockid != CLOCK_MONOTONIC &&
1150 	     clockid != CLOCK_REALTIME &&
1151 	     clockid != CLOCK_BOOTTIME))
1152 		return -EINVAL;
1153 	__bpf_spin_lock_irqsave(&timer->lock);
1154 	t = timer->timer;
1155 	if (t) {
1156 		ret = -EBUSY;
1157 		goto out;
1158 	}
1159 	if (!atomic64_read(&map->usercnt)) {
1160 		/* maps with timers must be either held by user space
1161 		 * or pinned in bpffs.
1162 		 */
1163 		ret = -EPERM;
1164 		goto out;
1165 	}
1166 	/* allocate hrtimer via map_kmalloc to use memcg accounting */
1167 	t = bpf_map_kmalloc_node(map, sizeof(*t), GFP_ATOMIC, map->numa_node);
1168 	if (!t) {
1169 		ret = -ENOMEM;
1170 		goto out;
1171 	}
1172 	t->value = (void *)timer - map->timer_off;
1173 	t->map = map;
1174 	t->prog = NULL;
1175 	rcu_assign_pointer(t->callback_fn, NULL);
1176 	hrtimer_init(&t->timer, clockid, HRTIMER_MODE_REL_SOFT);
1177 	t->timer.function = bpf_timer_cb;
1178 	timer->timer = t;
1179 out:
1180 	__bpf_spin_unlock_irqrestore(&timer->lock);
1181 	return ret;
1182 }
1183 
1184 static const struct bpf_func_proto bpf_timer_init_proto = {
1185 	.func		= bpf_timer_init,
1186 	.gpl_only	= true,
1187 	.ret_type	= RET_INTEGER,
1188 	.arg1_type	= ARG_PTR_TO_TIMER,
1189 	.arg2_type	= ARG_CONST_MAP_PTR,
1190 	.arg3_type	= ARG_ANYTHING,
1191 };
1192 
BPF_CALL_3(bpf_timer_set_callback,struct bpf_timer_kern *,timer,void *,callback_fn,struct bpf_prog_aux *,aux)1193 BPF_CALL_3(bpf_timer_set_callback, struct bpf_timer_kern *, timer, void *, callback_fn,
1194 	   struct bpf_prog_aux *, aux)
1195 {
1196 	struct bpf_prog *prev, *prog = aux->prog;
1197 	struct bpf_hrtimer *t;
1198 	int ret = 0;
1199 
1200 	if (in_nmi())
1201 		return -EOPNOTSUPP;
1202 	__bpf_spin_lock_irqsave(&timer->lock);
1203 	t = timer->timer;
1204 	if (!t) {
1205 		ret = -EINVAL;
1206 		goto out;
1207 	}
1208 	if (!atomic64_read(&t->map->usercnt)) {
1209 		/* maps with timers must be either held by user space
1210 		 * or pinned in bpffs. Otherwise timer might still be
1211 		 * running even when bpf prog is detached and user space
1212 		 * is gone, since map_release_uref won't ever be called.
1213 		 */
1214 		ret = -EPERM;
1215 		goto out;
1216 	}
1217 	prev = t->prog;
1218 	if (prev != prog) {
1219 		/* Bump prog refcnt once. Every bpf_timer_set_callback()
1220 		 * can pick different callback_fn-s within the same prog.
1221 		 */
1222 		prog = bpf_prog_inc_not_zero(prog);
1223 		if (IS_ERR(prog)) {
1224 			ret = PTR_ERR(prog);
1225 			goto out;
1226 		}
1227 		if (prev)
1228 			/* Drop prev prog refcnt when swapping with new prog */
1229 			bpf_prog_put(prev);
1230 		t->prog = prog;
1231 	}
1232 	rcu_assign_pointer(t->callback_fn, callback_fn);
1233 out:
1234 	__bpf_spin_unlock_irqrestore(&timer->lock);
1235 	return ret;
1236 }
1237 
1238 static const struct bpf_func_proto bpf_timer_set_callback_proto = {
1239 	.func		= bpf_timer_set_callback,
1240 	.gpl_only	= true,
1241 	.ret_type	= RET_INTEGER,
1242 	.arg1_type	= ARG_PTR_TO_TIMER,
1243 	.arg2_type	= ARG_PTR_TO_FUNC,
1244 };
1245 
BPF_CALL_3(bpf_timer_start,struct bpf_timer_kern *,timer,u64,nsecs,u64,flags)1246 BPF_CALL_3(bpf_timer_start, struct bpf_timer_kern *, timer, u64, nsecs, u64, flags)
1247 {
1248 	struct bpf_hrtimer *t;
1249 	int ret = 0;
1250 
1251 	if (in_nmi())
1252 		return -EOPNOTSUPP;
1253 	if (flags)
1254 		return -EINVAL;
1255 	__bpf_spin_lock_irqsave(&timer->lock);
1256 	t = timer->timer;
1257 	if (!t || !t->prog) {
1258 		ret = -EINVAL;
1259 		goto out;
1260 	}
1261 	hrtimer_start(&t->timer, ns_to_ktime(nsecs), HRTIMER_MODE_REL_SOFT);
1262 out:
1263 	__bpf_spin_unlock_irqrestore(&timer->lock);
1264 	return ret;
1265 }
1266 
1267 static const struct bpf_func_proto bpf_timer_start_proto = {
1268 	.func		= bpf_timer_start,
1269 	.gpl_only	= true,
1270 	.ret_type	= RET_INTEGER,
1271 	.arg1_type	= ARG_PTR_TO_TIMER,
1272 	.arg2_type	= ARG_ANYTHING,
1273 	.arg3_type	= ARG_ANYTHING,
1274 };
1275 
drop_prog_refcnt(struct bpf_hrtimer * t)1276 static void drop_prog_refcnt(struct bpf_hrtimer *t)
1277 {
1278 	struct bpf_prog *prog = t->prog;
1279 
1280 	if (prog) {
1281 		bpf_prog_put(prog);
1282 		t->prog = NULL;
1283 		rcu_assign_pointer(t->callback_fn, NULL);
1284 	}
1285 }
1286 
BPF_CALL_1(bpf_timer_cancel,struct bpf_timer_kern *,timer)1287 BPF_CALL_1(bpf_timer_cancel, struct bpf_timer_kern *, timer)
1288 {
1289 	struct bpf_hrtimer *t;
1290 	int ret = 0;
1291 
1292 	if (in_nmi())
1293 		return -EOPNOTSUPP;
1294 	__bpf_spin_lock_irqsave(&timer->lock);
1295 	t = timer->timer;
1296 	if (!t) {
1297 		ret = -EINVAL;
1298 		goto out;
1299 	}
1300 	if (this_cpu_read(hrtimer_running) == t) {
1301 		/* If bpf callback_fn is trying to bpf_timer_cancel()
1302 		 * its own timer the hrtimer_cancel() will deadlock
1303 		 * since it waits for callback_fn to finish
1304 		 */
1305 		ret = -EDEADLK;
1306 		goto out;
1307 	}
1308 	drop_prog_refcnt(t);
1309 out:
1310 	__bpf_spin_unlock_irqrestore(&timer->lock);
1311 	/* Cancel the timer and wait for associated callback to finish
1312 	 * if it was running.
1313 	 */
1314 	ret = ret ?: hrtimer_cancel(&t->timer);
1315 	return ret;
1316 }
1317 
1318 static const struct bpf_func_proto bpf_timer_cancel_proto = {
1319 	.func		= bpf_timer_cancel,
1320 	.gpl_only	= true,
1321 	.ret_type	= RET_INTEGER,
1322 	.arg1_type	= ARG_PTR_TO_TIMER,
1323 };
1324 
1325 /* This function is called by map_delete/update_elem for individual element and
1326  * by ops->map_release_uref when the user space reference to a map reaches zero.
1327  */
bpf_timer_cancel_and_free(void * val)1328 void bpf_timer_cancel_and_free(void *val)
1329 {
1330 	struct bpf_timer_kern *timer = val;
1331 	struct bpf_hrtimer *t;
1332 
1333 	/* Performance optimization: read timer->timer without lock first. */
1334 	if (!READ_ONCE(timer->timer))
1335 		return;
1336 
1337 	__bpf_spin_lock_irqsave(&timer->lock);
1338 	/* re-read it under lock */
1339 	t = timer->timer;
1340 	if (!t)
1341 		goto out;
1342 	drop_prog_refcnt(t);
1343 	/* The subsequent bpf_timer_start/cancel() helpers won't be able to use
1344 	 * this timer, since it won't be initialized.
1345 	 */
1346 	timer->timer = NULL;
1347 out:
1348 	__bpf_spin_unlock_irqrestore(&timer->lock);
1349 	if (!t)
1350 		return;
1351 	/* Cancel the timer and wait for callback to complete if it was running.
1352 	 * If hrtimer_cancel() can be safely called it's safe to call kfree(t)
1353 	 * right after for both preallocated and non-preallocated maps.
1354 	 * The timer->timer = NULL was already done and no code path can
1355 	 * see address 't' anymore.
1356 	 *
1357 	 * Check that bpf_map_delete/update_elem() wasn't called from timer
1358 	 * callback_fn. In such case don't call hrtimer_cancel() (since it will
1359 	 * deadlock) and don't call hrtimer_try_to_cancel() (since it will just
1360 	 * return -1). Though callback_fn is still running on this cpu it's
1361 	 * safe to do kfree(t) because bpf_timer_cb() read everything it needed
1362 	 * from 't'. The bpf subprog callback_fn won't be able to access 't',
1363 	 * since timer->timer = NULL was already done. The timer will be
1364 	 * effectively cancelled because bpf_timer_cb() will return
1365 	 * HRTIMER_NORESTART.
1366 	 */
1367 	if (this_cpu_read(hrtimer_running) != t)
1368 		hrtimer_cancel(&t->timer);
1369 	kfree(t);
1370 }
1371 
BPF_CALL_2(bpf_kptr_xchg,void *,map_value,void *,ptr)1372 BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr)
1373 {
1374 	unsigned long *kptr = map_value;
1375 
1376 	return xchg(kptr, (unsigned long)ptr);
1377 }
1378 
1379 /* Unlike other PTR_TO_BTF_ID helpers the btf_id in bpf_kptr_xchg()
1380  * helper is determined dynamically by the verifier. Use BPF_PTR_POISON to
1381  * denote type that verifier will determine.
1382  */
1383 static const struct bpf_func_proto bpf_kptr_xchg_proto = {
1384 	.func         = bpf_kptr_xchg,
1385 	.gpl_only     = false,
1386 	.ret_type     = RET_PTR_TO_BTF_ID_OR_NULL,
1387 	.ret_btf_id   = BPF_PTR_POISON,
1388 	.arg1_type    = ARG_PTR_TO_KPTR,
1389 	.arg2_type    = ARG_PTR_TO_BTF_ID_OR_NULL | OBJ_RELEASE,
1390 	.arg2_btf_id  = BPF_PTR_POISON,
1391 };
1392 
1393 /* Since the upper 8 bits of dynptr->size is reserved, the
1394  * maximum supported size is 2^24 - 1.
1395  */
1396 #define DYNPTR_MAX_SIZE	((1UL << 24) - 1)
1397 #define DYNPTR_TYPE_SHIFT	28
1398 #define DYNPTR_SIZE_MASK	0xFFFFFF
1399 #define DYNPTR_RDONLY_BIT	BIT(31)
1400 
bpf_dynptr_is_rdonly(struct bpf_dynptr_kern * ptr)1401 static bool bpf_dynptr_is_rdonly(struct bpf_dynptr_kern *ptr)
1402 {
1403 	return ptr->size & DYNPTR_RDONLY_BIT;
1404 }
1405 
bpf_dynptr_set_type(struct bpf_dynptr_kern * ptr,enum bpf_dynptr_type type)1406 static void bpf_dynptr_set_type(struct bpf_dynptr_kern *ptr, enum bpf_dynptr_type type)
1407 {
1408 	ptr->size |= type << DYNPTR_TYPE_SHIFT;
1409 }
1410 
bpf_dynptr_get_size(struct bpf_dynptr_kern * ptr)1411 u32 bpf_dynptr_get_size(struct bpf_dynptr_kern *ptr)
1412 {
1413 	return ptr->size & DYNPTR_SIZE_MASK;
1414 }
1415 
bpf_dynptr_check_size(u32 size)1416 int bpf_dynptr_check_size(u32 size)
1417 {
1418 	return size > DYNPTR_MAX_SIZE ? -E2BIG : 0;
1419 }
1420 
bpf_dynptr_init(struct bpf_dynptr_kern * ptr,void * data,enum bpf_dynptr_type type,u32 offset,u32 size)1421 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
1422 		     enum bpf_dynptr_type type, u32 offset, u32 size)
1423 {
1424 	ptr->data = data;
1425 	ptr->offset = offset;
1426 	ptr->size = size;
1427 	bpf_dynptr_set_type(ptr, type);
1428 }
1429 
bpf_dynptr_set_null(struct bpf_dynptr_kern * ptr)1430 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
1431 {
1432 	memset(ptr, 0, sizeof(*ptr));
1433 }
1434 
bpf_dynptr_check_off_len(struct bpf_dynptr_kern * ptr,u32 offset,u32 len)1435 static int bpf_dynptr_check_off_len(struct bpf_dynptr_kern *ptr, u32 offset, u32 len)
1436 {
1437 	u32 size = bpf_dynptr_get_size(ptr);
1438 
1439 	if (len > size || offset > size - len)
1440 		return -E2BIG;
1441 
1442 	return 0;
1443 }
1444 
BPF_CALL_4(bpf_dynptr_from_mem,void *,data,u32,size,u64,flags,struct bpf_dynptr_kern *,ptr)1445 BPF_CALL_4(bpf_dynptr_from_mem, void *, data, u32, size, u64, flags, struct bpf_dynptr_kern *, ptr)
1446 {
1447 	int err;
1448 
1449 	BTF_TYPE_EMIT(struct bpf_dynptr);
1450 
1451 	err = bpf_dynptr_check_size(size);
1452 	if (err)
1453 		goto error;
1454 
1455 	/* flags is currently unsupported */
1456 	if (flags) {
1457 		err = -EINVAL;
1458 		goto error;
1459 	}
1460 
1461 	bpf_dynptr_init(ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, size);
1462 
1463 	return 0;
1464 
1465 error:
1466 	bpf_dynptr_set_null(ptr);
1467 	return err;
1468 }
1469 
1470 static const struct bpf_func_proto bpf_dynptr_from_mem_proto = {
1471 	.func		= bpf_dynptr_from_mem,
1472 	.gpl_only	= false,
1473 	.ret_type	= RET_INTEGER,
1474 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
1475 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
1476 	.arg3_type	= ARG_ANYTHING,
1477 	.arg4_type	= ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT,
1478 };
1479 
BPF_CALL_5(bpf_dynptr_read,void *,dst,u32,len,struct bpf_dynptr_kern *,src,u32,offset,u64,flags)1480 BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, struct bpf_dynptr_kern *, src,
1481 	   u32, offset, u64, flags)
1482 {
1483 	int err;
1484 
1485 	if (!src->data || flags)
1486 		return -EINVAL;
1487 
1488 	err = bpf_dynptr_check_off_len(src, offset, len);
1489 	if (err)
1490 		return err;
1491 
1492 	memcpy(dst, src->data + src->offset + offset, len);
1493 
1494 	return 0;
1495 }
1496 
1497 static const struct bpf_func_proto bpf_dynptr_read_proto = {
1498 	.func		= bpf_dynptr_read,
1499 	.gpl_only	= false,
1500 	.ret_type	= RET_INTEGER,
1501 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
1502 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
1503 	.arg3_type	= ARG_PTR_TO_DYNPTR,
1504 	.arg4_type	= ARG_ANYTHING,
1505 	.arg5_type	= ARG_ANYTHING,
1506 };
1507 
BPF_CALL_5(bpf_dynptr_write,struct bpf_dynptr_kern *,dst,u32,offset,void *,src,u32,len,u64,flags)1508 BPF_CALL_5(bpf_dynptr_write, struct bpf_dynptr_kern *, dst, u32, offset, void *, src,
1509 	   u32, len, u64, flags)
1510 {
1511 	int err;
1512 
1513 	if (!dst->data || flags || bpf_dynptr_is_rdonly(dst))
1514 		return -EINVAL;
1515 
1516 	err = bpf_dynptr_check_off_len(dst, offset, len);
1517 	if (err)
1518 		return err;
1519 
1520 	memcpy(dst->data + dst->offset + offset, src, len);
1521 
1522 	return 0;
1523 }
1524 
1525 static const struct bpf_func_proto bpf_dynptr_write_proto = {
1526 	.func		= bpf_dynptr_write,
1527 	.gpl_only	= false,
1528 	.ret_type	= RET_INTEGER,
1529 	.arg1_type	= ARG_PTR_TO_DYNPTR,
1530 	.arg2_type	= ARG_ANYTHING,
1531 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1532 	.arg4_type	= ARG_CONST_SIZE_OR_ZERO,
1533 	.arg5_type	= ARG_ANYTHING,
1534 };
1535 
BPF_CALL_3(bpf_dynptr_data,struct bpf_dynptr_kern *,ptr,u32,offset,u32,len)1536 BPF_CALL_3(bpf_dynptr_data, struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
1537 {
1538 	int err;
1539 
1540 	if (!ptr->data)
1541 		return 0;
1542 
1543 	err = bpf_dynptr_check_off_len(ptr, offset, len);
1544 	if (err)
1545 		return 0;
1546 
1547 	if (bpf_dynptr_is_rdonly(ptr))
1548 		return 0;
1549 
1550 	return (unsigned long)(ptr->data + ptr->offset + offset);
1551 }
1552 
1553 static const struct bpf_func_proto bpf_dynptr_data_proto = {
1554 	.func		= bpf_dynptr_data,
1555 	.gpl_only	= false,
1556 	.ret_type	= RET_PTR_TO_DYNPTR_MEM_OR_NULL,
1557 	.arg1_type	= ARG_PTR_TO_DYNPTR,
1558 	.arg2_type	= ARG_ANYTHING,
1559 	.arg3_type	= ARG_CONST_ALLOC_SIZE_OR_ZERO,
1560 };
1561 
1562 const struct bpf_func_proto bpf_get_current_task_proto __weak;
1563 const struct bpf_func_proto bpf_get_current_task_btf_proto __weak;
1564 const struct bpf_func_proto bpf_probe_read_user_proto __weak;
1565 const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;
1566 const struct bpf_func_proto bpf_probe_read_kernel_proto __weak;
1567 const struct bpf_func_proto bpf_probe_read_kernel_str_proto __weak;
1568 const struct bpf_func_proto bpf_task_pt_regs_proto __weak;
1569 
1570 const struct bpf_func_proto *
bpf_base_func_proto(enum bpf_func_id func_id)1571 bpf_base_func_proto(enum bpf_func_id func_id)
1572 {
1573 	switch (func_id) {
1574 	case BPF_FUNC_map_lookup_elem:
1575 		return &bpf_map_lookup_elem_proto;
1576 	case BPF_FUNC_map_update_elem:
1577 		return &bpf_map_update_elem_proto;
1578 	case BPF_FUNC_map_delete_elem:
1579 		return &bpf_map_delete_elem_proto;
1580 	case BPF_FUNC_map_push_elem:
1581 		return &bpf_map_push_elem_proto;
1582 	case BPF_FUNC_map_pop_elem:
1583 		return &bpf_map_pop_elem_proto;
1584 	case BPF_FUNC_map_peek_elem:
1585 		return &bpf_map_peek_elem_proto;
1586 	case BPF_FUNC_map_lookup_percpu_elem:
1587 		return &bpf_map_lookup_percpu_elem_proto;
1588 	case BPF_FUNC_get_prandom_u32:
1589 		return &bpf_get_prandom_u32_proto;
1590 	case BPF_FUNC_get_smp_processor_id:
1591 		return &bpf_get_raw_smp_processor_id_proto;
1592 	case BPF_FUNC_get_numa_node_id:
1593 		return &bpf_get_numa_node_id_proto;
1594 	case BPF_FUNC_tail_call:
1595 		return &bpf_tail_call_proto;
1596 	case BPF_FUNC_ktime_get_ns:
1597 		return &bpf_ktime_get_ns_proto;
1598 	case BPF_FUNC_ktime_get_boot_ns:
1599 		return &bpf_ktime_get_boot_ns_proto;
1600 	case BPF_FUNC_ktime_get_tai_ns:
1601 		return &bpf_ktime_get_tai_ns_proto;
1602 	case BPF_FUNC_ringbuf_output:
1603 		return &bpf_ringbuf_output_proto;
1604 	case BPF_FUNC_ringbuf_reserve:
1605 		return &bpf_ringbuf_reserve_proto;
1606 	case BPF_FUNC_ringbuf_submit:
1607 		return &bpf_ringbuf_submit_proto;
1608 	case BPF_FUNC_ringbuf_discard:
1609 		return &bpf_ringbuf_discard_proto;
1610 	case BPF_FUNC_ringbuf_query:
1611 		return &bpf_ringbuf_query_proto;
1612 	case BPF_FUNC_strncmp:
1613 		return &bpf_strncmp_proto;
1614 	case BPF_FUNC_strtol:
1615 		return &bpf_strtol_proto;
1616 	case BPF_FUNC_strtoul:
1617 		return &bpf_strtoul_proto;
1618 	default:
1619 		break;
1620 	}
1621 
1622 	if (!bpf_capable())
1623 		return NULL;
1624 
1625 	switch (func_id) {
1626 	case BPF_FUNC_spin_lock:
1627 		return &bpf_spin_lock_proto;
1628 	case BPF_FUNC_spin_unlock:
1629 		return &bpf_spin_unlock_proto;
1630 	case BPF_FUNC_jiffies64:
1631 		return &bpf_jiffies64_proto;
1632 	case BPF_FUNC_per_cpu_ptr:
1633 		return &bpf_per_cpu_ptr_proto;
1634 	case BPF_FUNC_this_cpu_ptr:
1635 		return &bpf_this_cpu_ptr_proto;
1636 	case BPF_FUNC_timer_init:
1637 		return &bpf_timer_init_proto;
1638 	case BPF_FUNC_timer_set_callback:
1639 		return &bpf_timer_set_callback_proto;
1640 	case BPF_FUNC_timer_start:
1641 		return &bpf_timer_start_proto;
1642 	case BPF_FUNC_timer_cancel:
1643 		return &bpf_timer_cancel_proto;
1644 	case BPF_FUNC_kptr_xchg:
1645 		return &bpf_kptr_xchg_proto;
1646 	case BPF_FUNC_for_each_map_elem:
1647 		return &bpf_for_each_map_elem_proto;
1648 	case BPF_FUNC_loop:
1649 		return &bpf_loop_proto;
1650 	case BPF_FUNC_user_ringbuf_drain:
1651 		return &bpf_user_ringbuf_drain_proto;
1652 	case BPF_FUNC_ringbuf_reserve_dynptr:
1653 		return &bpf_ringbuf_reserve_dynptr_proto;
1654 	case BPF_FUNC_ringbuf_submit_dynptr:
1655 		return &bpf_ringbuf_submit_dynptr_proto;
1656 	case BPF_FUNC_ringbuf_discard_dynptr:
1657 		return &bpf_ringbuf_discard_dynptr_proto;
1658 	case BPF_FUNC_dynptr_from_mem:
1659 		return &bpf_dynptr_from_mem_proto;
1660 	case BPF_FUNC_dynptr_read:
1661 		return &bpf_dynptr_read_proto;
1662 	case BPF_FUNC_dynptr_write:
1663 		return &bpf_dynptr_write_proto;
1664 	case BPF_FUNC_dynptr_data:
1665 		return &bpf_dynptr_data_proto;
1666 	default:
1667 		break;
1668 	}
1669 
1670 	if (!perfmon_capable())
1671 		return NULL;
1672 
1673 	switch (func_id) {
1674 	case BPF_FUNC_trace_printk:
1675 		return bpf_get_trace_printk_proto();
1676 	case BPF_FUNC_get_current_task:
1677 		return &bpf_get_current_task_proto;
1678 	case BPF_FUNC_get_current_task_btf:
1679 		return &bpf_get_current_task_btf_proto;
1680 	case BPF_FUNC_probe_read_user:
1681 		return &bpf_probe_read_user_proto;
1682 	case BPF_FUNC_probe_read_kernel:
1683 		return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1684 		       NULL : &bpf_probe_read_kernel_proto;
1685 	case BPF_FUNC_probe_read_user_str:
1686 		return &bpf_probe_read_user_str_proto;
1687 	case BPF_FUNC_probe_read_kernel_str:
1688 		return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1689 		       NULL : &bpf_probe_read_kernel_str_proto;
1690 	case BPF_FUNC_snprintf_btf:
1691 		return &bpf_snprintf_btf_proto;
1692 	case BPF_FUNC_snprintf:
1693 		return &bpf_snprintf_proto;
1694 	case BPF_FUNC_task_pt_regs:
1695 		return &bpf_task_pt_regs_proto;
1696 	case BPF_FUNC_trace_vprintk:
1697 		return bpf_get_trace_vprintk_proto();
1698 	default:
1699 		return NULL;
1700 	}
1701 }
1702 
1703 BTF_SET8_START(tracing_btf_ids)
1704 #ifdef CONFIG_KEXEC_CORE
1705 BTF_ID_FLAGS(func, crash_kexec, KF_DESTRUCTIVE)
1706 #endif
1707 BTF_SET8_END(tracing_btf_ids)
1708 
1709 static const struct btf_kfunc_id_set tracing_kfunc_set = {
1710 	.owner = THIS_MODULE,
1711 	.set   = &tracing_btf_ids,
1712 };
1713 
kfunc_init(void)1714 static int __init kfunc_init(void)
1715 {
1716 	return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &tracing_kfunc_set);
1717 }
1718 
1719 late_initcall(kfunc_init);
1720