1 use crate::arch::{ 2 asm::csr::{ 3 CSR_SCAUSE, CSR_SEPC, CSR_SSCRATCH, CSR_SSTATUS, CSR_STVAL, SR_FS_VS, SR_SPP, SR_SUM, 4 }, 5 cpu::LocalContext, 6 interrupt::TrapFrame, 7 }; 8 use core::arch::asm; 9 use kdepends::memoffset::offset_of; 10 11 /// 保存x6-x31寄存器 12 macro_rules! save_from_x6_to_x31 { 13 () => { 14 concat!( 15 " 16 sd x6, {off_t1}(sp) 17 sd x7, {off_t2}(sp) 18 sd x8, {off_s0}(sp) 19 sd x9, {off_s1}(sp) 20 sd x10, {off_a0}(sp) 21 sd x11, {off_a1}(sp) 22 sd x12, {off_a2}(sp) 23 sd x13, {off_a3}(sp) 24 sd x14, {off_a4}(sp) 25 sd x15, {off_a5}(sp) 26 sd x16, {off_a6}(sp) 27 sd x17, {off_a7}(sp) 28 sd x18, {off_s2}(sp) 29 sd x19, {off_s3}(sp) 30 sd x20, {off_s4}(sp) 31 sd x21, {off_s5}(sp) 32 sd x22, {off_s6}(sp) 33 sd x23, {off_s7}(sp) 34 sd x24, {off_s8}(sp) 35 sd x25, {off_s9}(sp) 36 sd x26, {off_s10}(sp) 37 sd x27, {off_s11}(sp) 38 sd x28, {off_t3}(sp) 39 sd x29, {off_t4}(sp) 40 sd x30, {off_t5}(sp) 41 sd x31, {off_t6}(sp) 42 43 " 44 ) 45 }; 46 } 47 48 macro_rules! restore_from_x6_to_x31 { 49 () => { 50 concat!(" 51 52 ld x6, {off_t1}(sp) 53 ld x7, {off_t2}(sp) 54 ld x8, {off_s0}(sp) 55 ld x9, {off_s1}(sp) 56 ld x10, {off_a0}(sp) 57 ld x11, {off_a1}(sp) 58 ld x12, {off_a2}(sp) 59 ld x13, {off_a3}(sp) 60 ld x14, {off_a4}(sp) 61 ld x15, {off_a5}(sp) 62 ld x16, {off_a6}(sp) 63 ld x17, {off_a7}(sp) 64 ld x18, {off_s2}(sp) 65 ld x19, {off_s3}(sp) 66 ld x20, {off_s4}(sp) 67 ld x21, {off_s5}(sp) 68 ld x22, {off_s6}(sp) 69 ld x23, {off_s7}(sp) 70 ld x24, {off_s8}(sp) 71 ld x25, {off_s9}(sp) 72 ld x26, {off_s10}(sp) 73 ld x27, {off_s11}(sp) 74 ld x28, {off_t3}(sp) 75 ld x29, {off_t4}(sp) 76 ld x30, {off_t5}(sp) 77 ld x31, {off_t6}(sp) 78 ") 79 }; 80 } 81 82 /// Riscv64中断处理入口 83 #[naked] 84 #[no_mangle] 85 #[repr(align(4))] 86 pub unsafe extern "C" fn handle_exception() -> ! { 87 asm!( 88 concat!(" 89 /* 90 * If coming from userspace, preserve the user thread pointer and load 91 * the kernel thread pointer. If we came from the kernel, the scratch 92 * register will contain 0, and we should continue on the current TP. 93 */ 94 95 csrrw tp, {csr_scratch}, tp 96 bnez tp, _save_context 97 98 /* 从内核态进入中断 */ 99 j {_restore_kernel_tpsp} 100 "), 101 csr_scratch = const CSR_SSCRATCH, 102 _restore_kernel_tpsp = sym _restore_kernel_tpsp, 103 options(noreturn), 104 ) 105 } 106 107 #[naked] 108 #[no_mangle] 109 unsafe extern "C" fn _restore_kernel_tpsp() -> ! { 110 asm!( 111 concat!(" 112 // 这次是从内核态进入中断 113 // 从sscratch寄存器加载当前cpu的上下文 114 csrr tp, {csr_scratch} 115 116 // 把当前的sp寄存器的值保存到当前cpu的上下文的kernel_sp字段 117 sd sp, {lc_off_kernel_sp}(tp) 118 119 j {_save_context} 120 "), 121 csr_scratch = const CSR_SSCRATCH, 122 lc_off_kernel_sp = const offset_of!(LocalContext, kernel_sp), 123 _save_context = sym _save_context, 124 125 options(noreturn), 126 ) 127 } 128 129 #[naked] 130 #[no_mangle] 131 unsafe extern "C" fn _save_context() -> ! { 132 asm!( 133 concat!(" 134 135 136 // 保存当前cpu的上下文 137 138 // 保存用户sp 139 sd sp, {lc_off_user_sp}(tp) 140 // 加载内核sp 141 ld sp, {lc_off_kernel_sp}(tp) 142 143 addi sp, sp, -{trap_frame_size_on_stack} 144 sd x1, {off_ra}(sp) 145 sd x3, {off_gp}(sp) 146 sd x5, {off_t0}(sp) 147 ", 148 save_from_x6_to_x31!(), 149 " 150 /* 151 * Disable user-mode memory access as it should only be set in the 152 * actual user copy routines. 153 * 154 * Disable the FPU/Vector to detect illegal usage of floating point 155 * or vector in kernel space. 156 */ 157 158 li t0, {sr_sum_and_fsvs} 159 160 ld s0, {lc_off_user_sp}(tp) 161 csrrc s1, {csr_status}, t0 162 csrr s2, {csr_epc} 163 csrr s3, {csr_tval} 164 csrr s4, {csr_cause} 165 csrr s5, {csr_scratch} 166 sd s0, {off_sp}(sp) 167 sd s1, {off_status}(sp) 168 sd s2, {off_epc}(sp) 169 sd s3, {off_badaddr}(sp) 170 sd s4, {off_cause}(sp) 171 sd s5, {off_tp}(sp) 172 173 /* 174 * Set the scratch register to 0, so that if a recursive exception 175 * occurs, the exception vector knows it came from the kernel 176 */ 177 178 csrw {csr_scratch}, x0 179 180 /* Load the global pointer */ 181 // linux 加载了global pointer,但是我们暂时没有用到 182 183 // .option push 184 // .option norelax 185 // la gp, __global_pointer$ 186 // .option pop 187 188 mv a0, sp 189 la ra, ret_from_exception 190 191 tail riscv64_do_irq 192 " 193 ), 194 195 lc_off_user_sp = const offset_of!(LocalContext, user_sp), 196 lc_off_kernel_sp = const offset_of!(LocalContext, kernel_sp), 197 trap_frame_size_on_stack = const TrapFrame::SIZE_ON_STACK, 198 off_ra = const offset_of!(TrapFrame, ra), 199 off_gp = const offset_of!(TrapFrame, gp), 200 off_t0 = const offset_of!(TrapFrame, t0), 201 off_t1 = const offset_of!(TrapFrame, t1), 202 off_t2 = const offset_of!(TrapFrame, t2), 203 off_s0 = const offset_of!(TrapFrame, s0), 204 off_s1 = const offset_of!(TrapFrame, s1), 205 off_a0 = const offset_of!(TrapFrame, a0), 206 off_a1 = const offset_of!(TrapFrame, a1), 207 off_a2 = const offset_of!(TrapFrame, a2), 208 off_a3 = const offset_of!(TrapFrame, a3), 209 off_a4 = const offset_of!(TrapFrame, a4), 210 off_a5 = const offset_of!(TrapFrame, a5), 211 off_a6 = const offset_of!(TrapFrame, a6), 212 off_a7 = const offset_of!(TrapFrame, a7), 213 off_s2 = const offset_of!(TrapFrame, s2), 214 off_s3 = const offset_of!(TrapFrame, s3), 215 off_s4 = const offset_of!(TrapFrame, s4), 216 off_s5 = const offset_of!(TrapFrame, s5), 217 off_s6 = const offset_of!(TrapFrame, s6), 218 off_s7 = const offset_of!(TrapFrame, s7), 219 off_s8 = const offset_of!(TrapFrame, s8), 220 off_s9 = const offset_of!(TrapFrame, s9), 221 off_s10 = const offset_of!(TrapFrame, s10), 222 off_s11 = const offset_of!(TrapFrame, s11), 223 off_t3 = const offset_of!(TrapFrame, t3), 224 off_t4 = const offset_of!(TrapFrame, t4), 225 off_t5 = const offset_of!(TrapFrame, t5), 226 off_t6 = const offset_of!(TrapFrame, t6), 227 off_sp = const offset_of!(TrapFrame, sp), 228 off_status = const offset_of!(TrapFrame, status), 229 off_badaddr = const offset_of!(TrapFrame, badaddr), 230 off_cause = const offset_of!(TrapFrame, cause), 231 off_tp = const offset_of!(TrapFrame, tp), 232 off_epc = const offset_of!(TrapFrame, epc), 233 sr_sum_and_fsvs = const (SR_FS_VS | SR_SUM), 234 csr_status = const CSR_SSTATUS, 235 csr_epc = const CSR_SEPC, 236 csr_tval = const CSR_STVAL, 237 csr_cause = const CSR_SCAUSE, 238 csr_scratch = const CSR_SSCRATCH, 239 options(noreturn), 240 ) 241 } 242 243 #[naked] 244 #[no_mangle] 245 pub unsafe extern "C" fn ret_from_exception() -> ! { 246 asm!( 247 concat!(" 248 ld s0, {off_status}(sp) 249 andi s0, s0, {sr_spp} 250 251 bnez s0, 3f 252 253 // Save unwound kernel stack pointer in thread_info 254 addi s0, sp, {trap_frame_size_on_stack} 255 sd s0, {lc_off_kernel_sp}(tp) 256 257 /* 258 * Save TP into the scratch register , so we can find the kernel data 259 * structures again. 260 */ 261 csrw {csr_scratch}, tp 262 3: 263 264 ld a0, {off_status}(sp) 265 266 ld a2, {off_epc}(sp) 267 sc.d x0, a2, {off_epc}(sp) 268 269 csrw {csr_status}, a0 270 csrw {csr_epc}, a2 271 272 ld x1, {off_ra}(sp) 273 ld x3, {off_gp}(sp) 274 ld x4, {off_tp}(sp) 275 ld x5, {off_t0}(sp) 276 277 ", 278 restore_from_x6_to_x31!(), 279 " 280 ld x2, {off_sp}(sp) 281 282 sret 283 " 284 ), 285 off_status = const offset_of!(TrapFrame, status), 286 sr_spp = const SR_SPP, 287 trap_frame_size_on_stack = const TrapFrame::SIZE_ON_STACK, 288 lc_off_kernel_sp = const offset_of!(LocalContext, kernel_sp), 289 csr_scratch = const CSR_SSCRATCH, 290 csr_status = const CSR_SSTATUS, 291 csr_epc = const CSR_SEPC, 292 off_ra = const offset_of!(TrapFrame, ra), 293 off_gp = const offset_of!(TrapFrame, gp), 294 off_t0 = const offset_of!(TrapFrame, t0), 295 off_t1 = const offset_of!(TrapFrame, t1), 296 off_t2 = const offset_of!(TrapFrame, t2), 297 off_s0 = const offset_of!(TrapFrame, s0), 298 off_s1 = const offset_of!(TrapFrame, s1), 299 off_a0 = const offset_of!(TrapFrame, a0), 300 off_a1 = const offset_of!(TrapFrame, a1), 301 off_a2 = const offset_of!(TrapFrame, a2), 302 off_a3 = const offset_of!(TrapFrame, a3), 303 off_a4 = const offset_of!(TrapFrame, a4), 304 off_a5 = const offset_of!(TrapFrame, a5), 305 off_a6 = const offset_of!(TrapFrame, a6), 306 off_a7 = const offset_of!(TrapFrame, a7), 307 off_s2 = const offset_of!(TrapFrame, s2), 308 off_s3 = const offset_of!(TrapFrame, s3), 309 off_s4 = const offset_of!(TrapFrame, s4), 310 off_s5 = const offset_of!(TrapFrame, s5), 311 off_s6 = const offset_of!(TrapFrame, s6), 312 off_s7 = const offset_of!(TrapFrame, s7), 313 off_s8 = const offset_of!(TrapFrame, s8), 314 off_s9 = const offset_of!(TrapFrame, s9), 315 off_s10 = const offset_of!(TrapFrame, s10), 316 off_s11 = const offset_of!(TrapFrame, s11), 317 off_t3 = const offset_of!(TrapFrame, t3), 318 off_t4 = const offset_of!(TrapFrame, t4), 319 off_t5 = const offset_of!(TrapFrame, t5), 320 off_t6 = const offset_of!(TrapFrame, t6), 321 off_sp = const offset_of!(TrapFrame, sp), 322 off_tp = const offset_of!(TrapFrame, tp), 323 off_epc = const offset_of!(TrapFrame, epc), 324 325 options(noreturn), 326 ) 327 } 328