1 use crate::arch::{
2 asm::csr::{
3 CSR_SCAUSE, CSR_SEPC, CSR_SSCRATCH, CSR_SSTATUS, CSR_STVAL, SR_FS_VS, SR_SPP, SR_SUM,
4 },
5 cpu::LocalContext,
6 interrupt::TrapFrame,
7 };
8 use core::arch::asm;
9 use kdepends::memoffset::offset_of;
10
11 /// 保存x6-x31寄存器
12 macro_rules! save_from_x6_to_x31 {
13 () => {
14 concat!(
15 "
16 sd x6, {off_t1}(sp)
17 sd x7, {off_t2}(sp)
18 sd x8, {off_s0}(sp)
19 sd x9, {off_s1}(sp)
20 sd x10, {off_a0}(sp)
21 sd x11, {off_a1}(sp)
22 sd x12, {off_a2}(sp)
23 sd x13, {off_a3}(sp)
24 sd x14, {off_a4}(sp)
25 sd x15, {off_a5}(sp)
26 sd x16, {off_a6}(sp)
27 sd x17, {off_a7}(sp)
28 sd x18, {off_s2}(sp)
29 sd x19, {off_s3}(sp)
30 sd x20, {off_s4}(sp)
31 sd x21, {off_s5}(sp)
32 sd x22, {off_s6}(sp)
33 sd x23, {off_s7}(sp)
34 sd x24, {off_s8}(sp)
35 sd x25, {off_s9}(sp)
36 sd x26, {off_s10}(sp)
37 sd x27, {off_s11}(sp)
38 sd x28, {off_t3}(sp)
39 sd x29, {off_t4}(sp)
40 sd x30, {off_t5}(sp)
41 sd x31, {off_t6}(sp)
42
43 "
44 )
45 };
46 }
47
48 macro_rules! restore_from_x6_to_x31 {
49 () => {
50 concat!("
51
52 ld x6, {off_t1}(sp)
53 ld x7, {off_t2}(sp)
54 ld x8, {off_s0}(sp)
55 ld x9, {off_s1}(sp)
56 ld x10, {off_a0}(sp)
57 ld x11, {off_a1}(sp)
58 ld x12, {off_a2}(sp)
59 ld x13, {off_a3}(sp)
60 ld x14, {off_a4}(sp)
61 ld x15, {off_a5}(sp)
62 ld x16, {off_a6}(sp)
63 ld x17, {off_a7}(sp)
64 ld x18, {off_s2}(sp)
65 ld x19, {off_s3}(sp)
66 ld x20, {off_s4}(sp)
67 ld x21, {off_s5}(sp)
68 ld x22, {off_s6}(sp)
69 ld x23, {off_s7}(sp)
70 ld x24, {off_s8}(sp)
71 ld x25, {off_s9}(sp)
72 ld x26, {off_s10}(sp)
73 ld x27, {off_s11}(sp)
74 ld x28, {off_t3}(sp)
75 ld x29, {off_t4}(sp)
76 ld x30, {off_t5}(sp)
77 ld x31, {off_t6}(sp)
78 ")
79 };
80 }
81
82 /// Riscv64中断处理入口
83 #[naked]
84 #[no_mangle]
85 #[repr(align(4))]
handle_exception() -> !86 pub unsafe extern "C" fn handle_exception() -> ! {
87 asm!(
88 concat!("
89 /*
90 * If coming from userspace, preserve the user thread pointer and load
91 * the kernel thread pointer. If we came from the kernel, the scratch
92 * register will contain 0, and we should continue on the current TP.
93 */
94
95 csrrw tp, {csr_scratch}, tp
96 bnez tp, _save_context
97
98 /* 从内核态进入中断 */
99 j {_restore_kernel_tpsp}
100 "),
101 csr_scratch = const CSR_SSCRATCH,
102 _restore_kernel_tpsp = sym _restore_kernel_tpsp,
103 options(noreturn),
104 )
105 }
106
107 #[naked]
108 #[no_mangle]
_restore_kernel_tpsp() -> !109 unsafe extern "C" fn _restore_kernel_tpsp() -> ! {
110 asm!(
111 concat!("
112 // 这次是从内核态进入中断
113 // 从sscratch寄存器加载当前cpu的上下文
114 csrr tp, {csr_scratch}
115
116 // 把当前的sp寄存器的值保存到当前cpu的上下文的kernel_sp字段
117 sd sp, {lc_off_kernel_sp}(tp)
118
119 j {_save_context}
120 "),
121 csr_scratch = const CSR_SSCRATCH,
122 lc_off_kernel_sp = const offset_of!(LocalContext, kernel_sp),
123 _save_context = sym _save_context,
124
125 options(noreturn),
126 )
127 }
128
129 #[naked]
130 #[no_mangle]
_save_context() -> !131 unsafe extern "C" fn _save_context() -> ! {
132 asm!(
133 concat!("
134
135
136 // 保存当前cpu的上下文
137
138 // 保存用户sp
139 sd sp, {lc_off_user_sp}(tp)
140 // 加载内核sp
141 ld sp, {lc_off_kernel_sp}(tp)
142
143 addi sp, sp, -{trap_frame_size_on_stack}
144 sd x1, {off_ra}(sp)
145 sd x3, {off_gp}(sp)
146 sd x5, {off_t0}(sp)
147 ",
148 save_from_x6_to_x31!(),
149 "
150 /*
151 * Disable user-mode memory access as it should only be set in the
152 * actual user copy routines.
153 *
154 * Disable the FPU/Vector to detect illegal usage of floating point
155 * or vector in kernel space.
156 */
157
158 li t0, {sr_sum_and_fsvs}
159
160 ld s0, {lc_off_user_sp}(tp)
161 csrrc s1, {csr_status}, t0
162 csrr s2, {csr_epc}
163 csrr s3, {csr_tval}
164 csrr s4, {csr_cause}
165 csrr s5, {csr_scratch}
166 sd s0, {off_sp}(sp)
167 sd s1, {off_status}(sp)
168 sd s2, {off_epc}(sp)
169 sd s3, {off_badaddr}(sp)
170 sd s4, {off_cause}(sp)
171 sd s5, {off_tp}(sp)
172
173 /*
174 * Set the scratch register to 0, so that if a recursive exception
175 * occurs, the exception vector knows it came from the kernel
176 */
177
178 csrw {csr_scratch}, x0
179
180 /* Load the global pointer */
181 // linux 加载了global pointer,但是我们暂时没有用到
182
183 // .option push
184 // .option norelax
185 // la gp, __global_pointer$
186 // .option pop
187
188 mv a0, sp
189 la ra, ret_from_exception
190
191 tail riscv64_do_irq
192 "
193 ),
194
195 lc_off_user_sp = const offset_of!(LocalContext, user_sp),
196 lc_off_kernel_sp = const offset_of!(LocalContext, kernel_sp),
197 trap_frame_size_on_stack = const TrapFrame::SIZE_ON_STACK,
198 off_ra = const offset_of!(TrapFrame, ra),
199 off_gp = const offset_of!(TrapFrame, gp),
200 off_t0 = const offset_of!(TrapFrame, t0),
201 off_t1 = const offset_of!(TrapFrame, t1),
202 off_t2 = const offset_of!(TrapFrame, t2),
203 off_s0 = const offset_of!(TrapFrame, s0),
204 off_s1 = const offset_of!(TrapFrame, s1),
205 off_a0 = const offset_of!(TrapFrame, a0),
206 off_a1 = const offset_of!(TrapFrame, a1),
207 off_a2 = const offset_of!(TrapFrame, a2),
208 off_a3 = const offset_of!(TrapFrame, a3),
209 off_a4 = const offset_of!(TrapFrame, a4),
210 off_a5 = const offset_of!(TrapFrame, a5),
211 off_a6 = const offset_of!(TrapFrame, a6),
212 off_a7 = const offset_of!(TrapFrame, a7),
213 off_s2 = const offset_of!(TrapFrame, s2),
214 off_s3 = const offset_of!(TrapFrame, s3),
215 off_s4 = const offset_of!(TrapFrame, s4),
216 off_s5 = const offset_of!(TrapFrame, s5),
217 off_s6 = const offset_of!(TrapFrame, s6),
218 off_s7 = const offset_of!(TrapFrame, s7),
219 off_s8 = const offset_of!(TrapFrame, s8),
220 off_s9 = const offset_of!(TrapFrame, s9),
221 off_s10 = const offset_of!(TrapFrame, s10),
222 off_s11 = const offset_of!(TrapFrame, s11),
223 off_t3 = const offset_of!(TrapFrame, t3),
224 off_t4 = const offset_of!(TrapFrame, t4),
225 off_t5 = const offset_of!(TrapFrame, t5),
226 off_t6 = const offset_of!(TrapFrame, t6),
227 off_sp = const offset_of!(TrapFrame, sp),
228 off_status = const offset_of!(TrapFrame, status),
229 off_badaddr = const offset_of!(TrapFrame, badaddr),
230 off_cause = const offset_of!(TrapFrame, cause),
231 off_tp = const offset_of!(TrapFrame, tp),
232 off_epc = const offset_of!(TrapFrame, epc),
233 sr_sum_and_fsvs = const (SR_FS_VS | SR_SUM),
234 csr_status = const CSR_SSTATUS,
235 csr_epc = const CSR_SEPC,
236 csr_tval = const CSR_STVAL,
237 csr_cause = const CSR_SCAUSE,
238 csr_scratch = const CSR_SSCRATCH,
239 options(noreturn),
240 )
241 }
242
243 #[naked]
244 #[no_mangle]
ret_from_exception() -> !245 unsafe extern "C" fn ret_from_exception() -> ! {
246 asm!(
247 concat!("
248 ld s0, {off_status}(sp)
249 andi s0, s0, {sr_spp}
250
251 bnez s0, 3f
252
253 // Save unwound kernel stack pointer in thread_info
254 addi s0, sp, {trap_frame_size_on_stack}
255 sd s0, {lc_off_kernel_sp}(tp)
256
257 /*
258 * Save TP into the scratch register , so we can find the kernel data
259 * structures again.
260 */
261 csrw {csr_scratch}, tp
262 3:
263
264 ld a0, {off_status}(sp)
265
266 ld a2, {off_epc}(sp)
267 sc.d x0, a2, {off_epc}(sp)
268
269 csrw {csr_status}, a0
270 csrw {csr_epc}, a2
271
272 ld x1, {off_ra}(sp)
273 ld x3, {off_gp}(sp)
274 ld x4, {off_tp}(sp)
275 ld x5, {off_t0}(sp)
276
277 ",
278 restore_from_x6_to_x31!(),
279 "
280 ld x2, {off_sp}(sp)
281
282 sret
283 "
284 ),
285 off_status = const offset_of!(TrapFrame, status),
286 sr_spp = const SR_SPP,
287 trap_frame_size_on_stack = const TrapFrame::SIZE_ON_STACK,
288 lc_off_kernel_sp = const offset_of!(LocalContext, kernel_sp),
289 csr_scratch = const CSR_SSCRATCH,
290 csr_status = const CSR_SSTATUS,
291 csr_epc = const CSR_SEPC,
292 off_ra = const offset_of!(TrapFrame, ra),
293 off_gp = const offset_of!(TrapFrame, gp),
294 off_t0 = const offset_of!(TrapFrame, t0),
295 off_t1 = const offset_of!(TrapFrame, t1),
296 off_t2 = const offset_of!(TrapFrame, t2),
297 off_s0 = const offset_of!(TrapFrame, s0),
298 off_s1 = const offset_of!(TrapFrame, s1),
299 off_a0 = const offset_of!(TrapFrame, a0),
300 off_a1 = const offset_of!(TrapFrame, a1),
301 off_a2 = const offset_of!(TrapFrame, a2),
302 off_a3 = const offset_of!(TrapFrame, a3),
303 off_a4 = const offset_of!(TrapFrame, a4),
304 off_a5 = const offset_of!(TrapFrame, a5),
305 off_a6 = const offset_of!(TrapFrame, a6),
306 off_a7 = const offset_of!(TrapFrame, a7),
307 off_s2 = const offset_of!(TrapFrame, s2),
308 off_s3 = const offset_of!(TrapFrame, s3),
309 off_s4 = const offset_of!(TrapFrame, s4),
310 off_s5 = const offset_of!(TrapFrame, s5),
311 off_s6 = const offset_of!(TrapFrame, s6),
312 off_s7 = const offset_of!(TrapFrame, s7),
313 off_s8 = const offset_of!(TrapFrame, s8),
314 off_s9 = const offset_of!(TrapFrame, s9),
315 off_s10 = const offset_of!(TrapFrame, s10),
316 off_s11 = const offset_of!(TrapFrame, s11),
317 off_t3 = const offset_of!(TrapFrame, t3),
318 off_t4 = const offset_of!(TrapFrame, t4),
319 off_t5 = const offset_of!(TrapFrame, t5),
320 off_t6 = const offset_of!(TrapFrame, t6),
321 off_sp = const offset_of!(TrapFrame, sp),
322 off_tp = const offset_of!(TrapFrame, tp),
323 off_epc = const offset_of!(TrapFrame, epc),
324
325 options(noreturn),
326 )
327 }
328