1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_COOKIE_H
3 #define __LINUX_COOKIE_H
4 
5 #include <linux/atomic.h>
6 #include <linux/percpu.h>
7 #include <asm/local.h>
8 
9 struct pcpu_gen_cookie {
10 	local_t nesting;
11 	u64 last;
12 } __aligned(16);
13 
14 struct gen_cookie {
15 	struct pcpu_gen_cookie __percpu *local;
16 	atomic64_t forward_last ____cacheline_aligned_in_smp;
17 	atomic64_t reverse_last;
18 };
19 
20 #define COOKIE_LOCAL_BATCH	4096
21 
22 #define DEFINE_COOKIE(name)						\
23 	static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name);	\
24 	static struct gen_cookie name = {				\
25 		.local		= &__##name,				\
26 		.forward_last	= ATOMIC64_INIT(0),			\
27 		.reverse_last	= ATOMIC64_INIT(0),			\
28 	}
29 
gen_cookie_next(struct gen_cookie * gc)30 static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
31 {
32 	struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
33 	u64 val;
34 
35 	if (likely(local_inc_return(&local->nesting) == 1)) {
36 		val = local->last;
37 		if (__is_defined(CONFIG_SMP) &&
38 		    unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
39 			s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
40 						       &gc->forward_last);
41 			val = next - COOKIE_LOCAL_BATCH;
42 		}
43 		local->last = ++val;
44 	} else {
45 		val = atomic64_dec_return(&gc->reverse_last);
46 	}
47 	local_dec(&local->nesting);
48 	return val;
49 }
50 
51 #endif /* __LINUX_COOKIE_H */
52