1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4  */
5 #ifndef __ASM_PERCPU_H
6 #define __ASM_PERCPU_H
7 
8 #include <asm/cmpxchg.h>
9 #include <asm/loongarch.h>
10 
11 /*
12  * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
13  * the loading address of main kernel image, but far from where the modules are
14  * loaded. Tell the compiler this fact when using explicit relocs.
15  */
16 #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
17 #define PER_CPU_ATTRIBUTES    __attribute__((model("extreme")))
18 #endif
19 
20 /* Use r21 for fast access */
21 register unsigned long __my_cpu_offset __asm__("$r21");
22 
set_my_cpu_offset(unsigned long off)23 static inline void set_my_cpu_offset(unsigned long off)
24 {
25 	__my_cpu_offset = off;
26 	csr_write64(off, PERCPU_BASE_KS);
27 }
28 #define __my_cpu_offset __my_cpu_offset
29 
30 #define PERCPU_OP(op, asm_op, c_op)					\
31 static inline unsigned long __percpu_##op(void *ptr,			\
32 			unsigned long val, int size)			\
33 {									\
34 	unsigned long ret;						\
35 									\
36 	switch (size) {							\
37 	case 4:								\
38 		__asm__ __volatile__(					\
39 		"am"#asm_op".w"	" %[ret], %[val], %[ptr]	\n"		\
40 		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr)		\
41 		: [val] "r" (val));					\
42 		break;							\
43 	case 8:								\
44 		__asm__ __volatile__(					\
45 		"am"#asm_op".d" " %[ret], %[val], %[ptr]	\n"		\
46 		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr)		\
47 		: [val] "r" (val));					\
48 		break;							\
49 	default:							\
50 		ret = 0;						\
51 		BUILD_BUG();						\
52 	}								\
53 									\
54 	return ret c_op val;						\
55 }
56 
57 PERCPU_OP(add, add, +)
58 PERCPU_OP(and, and, &)
59 PERCPU_OP(or, or, |)
60 #undef PERCPU_OP
61 
__percpu_read(void * ptr,int size)62 static inline unsigned long __percpu_read(void *ptr, int size)
63 {
64 	unsigned long ret;
65 
66 	switch (size) {
67 	case 1:
68 		__asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr]	\n"
69 		: [ret] "=&r"(ret)
70 		: [ptr] "r"(ptr)
71 		: "memory");
72 		break;
73 	case 2:
74 		__asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr]	\n"
75 		: [ret] "=&r"(ret)
76 		: [ptr] "r"(ptr)
77 		: "memory");
78 		break;
79 	case 4:
80 		__asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr]	\n"
81 		: [ret] "=&r"(ret)
82 		: [ptr] "r"(ptr)
83 		: "memory");
84 		break;
85 	case 8:
86 		__asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr]	\n"
87 		: [ret] "=&r"(ret)
88 		: [ptr] "r"(ptr)
89 		: "memory");
90 		break;
91 	default:
92 		ret = 0;
93 		BUILD_BUG();
94 	}
95 
96 	return ret;
97 }
98 
__percpu_write(void * ptr,unsigned long val,int size)99 static inline void __percpu_write(void *ptr, unsigned long val, int size)
100 {
101 	switch (size) {
102 	case 1:
103 		__asm__ __volatile__("stx.b %[val], $r21, %[ptr]	\n"
104 		:
105 		: [val] "r" (val), [ptr] "r" (ptr)
106 		: "memory");
107 		break;
108 	case 2:
109 		__asm__ __volatile__("stx.h %[val], $r21, %[ptr]	\n"
110 		:
111 		: [val] "r" (val), [ptr] "r" (ptr)
112 		: "memory");
113 		break;
114 	case 4:
115 		__asm__ __volatile__("stx.w %[val], $r21, %[ptr]	\n"
116 		:
117 		: [val] "r" (val), [ptr] "r" (ptr)
118 		: "memory");
119 		break;
120 	case 8:
121 		__asm__ __volatile__("stx.d %[val], $r21, %[ptr]	\n"
122 		:
123 		: [val] "r" (val), [ptr] "r" (ptr)
124 		: "memory");
125 		break;
126 	default:
127 		BUILD_BUG();
128 	}
129 }
130 
__percpu_xchg(void * ptr,unsigned long val,int size)131 static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
132 						int size)
133 {
134 	switch (size) {
135 	case 1:
136 	case 2:
137 		return __xchg_small((volatile void *)ptr, val, size);
138 
139 	case 4:
140 		return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
141 
142 	case 8:
143 		return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
144 
145 	default:
146 		BUILD_BUG();
147 	}
148 
149 	return 0;
150 }
151 
152 /* this_cpu_cmpxchg */
153 #define _protect_cmpxchg_local(pcp, o, n)			\
154 ({								\
155 	typeof(*raw_cpu_ptr(&(pcp))) __ret;			\
156 	preempt_disable_notrace();				\
157 	__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);	\
158 	preempt_enable_notrace();				\
159 	__ret;							\
160 })
161 
162 #define _percpu_read(pcp)						\
163 ({									\
164 	typeof(pcp) __retval;						\
165 	__retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));	\
166 	__retval;							\
167 })
168 
169 #define _percpu_write(pcp, val)						\
170 do {									\
171 	__percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp));	\
172 } while (0)								\
173 
174 #define _pcp_protect(operation, pcp, val)			\
175 ({								\
176 	typeof(pcp) __retval;					\
177 	preempt_disable_notrace();				\
178 	__retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),	\
179 					  (val), sizeof(pcp));	\
180 	preempt_enable_notrace();				\
181 	__retval;						\
182 })
183 
184 #define _percpu_add(pcp, val) \
185 	_pcp_protect(__percpu_add, pcp, val)
186 
187 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
188 
189 #define _percpu_and(pcp, val) \
190 	_pcp_protect(__percpu_and, pcp, val)
191 
192 #define _percpu_or(pcp, val) \
193 	_pcp_protect(__percpu_or, pcp, val)
194 
195 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \
196 	_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
197 
198 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
199 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
200 
201 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
202 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
203 
204 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
205 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
206 
207 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
208 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
209 
210 #define this_cpu_read_1(pcp) _percpu_read(pcp)
211 #define this_cpu_read_2(pcp) _percpu_read(pcp)
212 #define this_cpu_read_4(pcp) _percpu_read(pcp)
213 #define this_cpu_read_8(pcp) _percpu_read(pcp)
214 
215 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
216 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
217 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
218 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
219 
220 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
221 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
222 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
223 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
224 
225 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
226 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
227 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
228 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
229 
230 #include <asm-generic/percpu.h>
231 
232 #endif /* __ASM_PERCPU_H */
233