1a17651b1SMemoryShore use core::{ 2a17651b1SMemoryShore intrinsics::{likely, unlikely}, 3a17651b1SMemoryShore panic, 4a17651b1SMemoryShore }; 5a17651b1SMemoryShore 6a17651b1SMemoryShore use alloc::sync::Arc; 72eab6dd7S曾俊 use log::error; 8a17651b1SMemoryShore use x86::{bits64::rflags::RFlags, controlregs::Cr4}; 9a17651b1SMemoryShore 10a17651b1SMemoryShore use crate::{ 11a17651b1SMemoryShore arch::{ 12a17651b1SMemoryShore interrupt::{trap::X86PfErrorCode, TrapFrame}, 13*cf7f801eSMemoryShore ipc::signal::{SigCode, Signal}, 14a17651b1SMemoryShore mm::{MemoryManagementArch, X86_64MMArch}, 15a17651b1SMemoryShore CurrentIrqArch, MMArch, 16a17651b1SMemoryShore }, 17a17651b1SMemoryShore exception::InterruptArch, 18*cf7f801eSMemoryShore ipc::signal_types::{SigInfo, SigType}, 19a17651b1SMemoryShore mm::{ 20a17651b1SMemoryShore fault::{FaultFlags, PageFaultHandler, PageFaultMessage}, 21a17651b1SMemoryShore ucontext::{AddressSpace, LockedVMA}, 22a17651b1SMemoryShore VirtAddr, VmFaultReason, VmFlags, 23a17651b1SMemoryShore }, 24*cf7f801eSMemoryShore process::ProcessManager, 25a17651b1SMemoryShore }; 26a17651b1SMemoryShore 27a17651b1SMemoryShore use super::LockedFrameAllocator; 28a17651b1SMemoryShore 29a17651b1SMemoryShore pub type PageMapper = 30a17651b1SMemoryShore crate::mm::page::PageMapper<crate::arch::x86_64::mm::X86_64MMArch, LockedFrameAllocator>; 31a17651b1SMemoryShore 32a17651b1SMemoryShore impl X86_64MMArch { vma_access_error(vma: Arc<LockedVMA>, error_code: X86PfErrorCode) -> bool33a17651b1SMemoryShore pub fn vma_access_error(vma: Arc<LockedVMA>, error_code: X86PfErrorCode) -> bool { 34*cf7f801eSMemoryShore let vm_flags = *vma.lock_irqsave().vm_flags(); 35a17651b1SMemoryShore let foreign = false; 36a17651b1SMemoryShore if error_code.contains(X86PfErrorCode::X86_PF_PK) { 37a17651b1SMemoryShore return true; 38a17651b1SMemoryShore } 39a17651b1SMemoryShore 40a17651b1SMemoryShore if unlikely(error_code.contains(X86PfErrorCode::X86_PF_SGX)) { 41a17651b1SMemoryShore return true; 42a17651b1SMemoryShore } 43a17651b1SMemoryShore 44a17651b1SMemoryShore if !Self::vma_access_permitted( 45a17651b1SMemoryShore vma.clone(), 46a17651b1SMemoryShore error_code.contains(X86PfErrorCode::X86_PF_WRITE), 47a17651b1SMemoryShore error_code.contains(X86PfErrorCode::X86_PF_INSTR), 48a17651b1SMemoryShore foreign, 49a17651b1SMemoryShore ) { 50a17651b1SMemoryShore return true; 51a17651b1SMemoryShore } 52a17651b1SMemoryShore 53a17651b1SMemoryShore if error_code.contains(X86PfErrorCode::X86_PF_WRITE) { 54a17651b1SMemoryShore if unlikely(!vm_flags.contains(VmFlags::VM_WRITE)) { 55a17651b1SMemoryShore return true; 56a17651b1SMemoryShore } 57a17651b1SMemoryShore return false; 58a17651b1SMemoryShore } 59a17651b1SMemoryShore 60a17651b1SMemoryShore if unlikely(error_code.contains(X86PfErrorCode::X86_PF_PROT)) { 61a17651b1SMemoryShore return true; 62a17651b1SMemoryShore } 63a17651b1SMemoryShore 64a17651b1SMemoryShore if unlikely(!vma.is_accessible()) { 65a17651b1SMemoryShore return true; 66a17651b1SMemoryShore } 67a17651b1SMemoryShore false 68a17651b1SMemoryShore } 69a17651b1SMemoryShore show_fault_oops( regs: &'static TrapFrame, error_code: X86PfErrorCode, address: VirtAddr, )70a17651b1SMemoryShore pub fn show_fault_oops( 71a17651b1SMemoryShore regs: &'static TrapFrame, 72a17651b1SMemoryShore error_code: X86PfErrorCode, 73a17651b1SMemoryShore address: VirtAddr, 74a17651b1SMemoryShore ) { 75a17651b1SMemoryShore let mapper = 76a17651b1SMemoryShore unsafe { PageMapper::current(crate::mm::PageTableKind::User, LockedFrameAllocator) }; 77a17651b1SMemoryShore if let Some(entry) = mapper.get_entry(address, 0) { 78a17651b1SMemoryShore if entry.present() { 79a17651b1SMemoryShore if !entry.flags().has_execute() { 802eab6dd7S曾俊 error!("kernel tried to execute NX-protected page - exploit attempt?"); 81a17651b1SMemoryShore } else if mapper.table().phys().data() & MMArch::ENTRY_FLAG_USER != 0 82a17651b1SMemoryShore && unsafe { x86::controlregs::cr4().contains(Cr4::CR4_ENABLE_SMEP) } 83a17651b1SMemoryShore { 842eab6dd7S曾俊 error!("unable to execute userspace code (SMEP?)"); 85a17651b1SMemoryShore } 86a17651b1SMemoryShore } 87a17651b1SMemoryShore } 88a17651b1SMemoryShore if address.data() < X86_64MMArch::PAGE_SIZE && !regs.is_from_user() { 892eab6dd7S曾俊 error!( 90a17651b1SMemoryShore "BUG: kernel NULL pointer dereference, address: {:#x}", 91a17651b1SMemoryShore address.data() 92a17651b1SMemoryShore ); 93a17651b1SMemoryShore } else { 942eab6dd7S曾俊 error!( 95a17651b1SMemoryShore "BUG: unable to handle page fault for address: {:#x}", 96a17651b1SMemoryShore address.data() 97a17651b1SMemoryShore ); 98a17651b1SMemoryShore } 99a17651b1SMemoryShore 1002eab6dd7S曾俊 error!( 101a17651b1SMemoryShore "#PF: {} {} in {} mode\n", 102a17651b1SMemoryShore if error_code.contains(X86PfErrorCode::X86_PF_USER) { 103a17651b1SMemoryShore "user" 104a17651b1SMemoryShore } else { 105a17651b1SMemoryShore "supervisor" 106a17651b1SMemoryShore }, 107a17651b1SMemoryShore if error_code.contains(X86PfErrorCode::X86_PF_INSTR) { 108a17651b1SMemoryShore "instruction fetch" 109a17651b1SMemoryShore } else if error_code.contains(X86PfErrorCode::X86_PF_WRITE) { 110a17651b1SMemoryShore "write access" 111a17651b1SMemoryShore } else { 112a17651b1SMemoryShore "read access" 113a17651b1SMemoryShore }, 114a17651b1SMemoryShore if regs.is_from_user() { 115a17651b1SMemoryShore "user" 116a17651b1SMemoryShore } else { 117a17651b1SMemoryShore "kernel" 118a17651b1SMemoryShore } 119a17651b1SMemoryShore ); 1202eab6dd7S曾俊 error!( 121a17651b1SMemoryShore "#PF: error_code({:#04x}) - {}\n", 122a17651b1SMemoryShore error_code, 123a17651b1SMemoryShore if !error_code.contains(X86PfErrorCode::X86_PF_PROT) { 124a17651b1SMemoryShore "not-present page" 125a17651b1SMemoryShore } else if error_code.contains(X86PfErrorCode::X86_PF_RSVD) { 126a17651b1SMemoryShore "reserved bit violation" 127a17651b1SMemoryShore } else if error_code.contains(X86PfErrorCode::X86_PF_PK) { 128a17651b1SMemoryShore "protection keys violation" 129a17651b1SMemoryShore } else { 130a17651b1SMemoryShore "permissions violation" 131a17651b1SMemoryShore } 132a17651b1SMemoryShore ); 133a17651b1SMemoryShore } 134a17651b1SMemoryShore page_fault_oops( regs: &'static TrapFrame, error_code: X86PfErrorCode, address: VirtAddr, )135a17651b1SMemoryShore pub fn page_fault_oops( 136a17651b1SMemoryShore regs: &'static TrapFrame, 137a17651b1SMemoryShore error_code: X86PfErrorCode, 138a17651b1SMemoryShore address: VirtAddr, 139a17651b1SMemoryShore ) { 140a17651b1SMemoryShore if regs.is_from_user() { 141a17651b1SMemoryShore Self::show_fault_oops(regs, error_code, address); 142a17651b1SMemoryShore } 143a17651b1SMemoryShore panic!() 144a17651b1SMemoryShore } 145a17651b1SMemoryShore 146a17651b1SMemoryShore /// 内核态缺页异常处理 147a17651b1SMemoryShore /// ## 参数 148a17651b1SMemoryShore /// 149a17651b1SMemoryShore /// - `regs`: 中断栈帧 150a17651b1SMemoryShore /// - `error_code`: 错误标志 151a17651b1SMemoryShore /// - `address`: 发生缺页异常的虚拟地址 do_kern_addr_fault( _regs: &'static TrapFrame, error_code: X86PfErrorCode, address: VirtAddr, )152a17651b1SMemoryShore pub fn do_kern_addr_fault( 153a17651b1SMemoryShore _regs: &'static TrapFrame, 154a17651b1SMemoryShore error_code: X86PfErrorCode, 155a17651b1SMemoryShore address: VirtAddr, 156a17651b1SMemoryShore ) { 157a17651b1SMemoryShore panic!( 158a17651b1SMemoryShore "do_kern_addr_fault has not yet been implemented, 159a17651b1SMemoryShore fault address: {:#x}, 160a17651b1SMemoryShore error_code: {:#b}, 161a17651b1SMemoryShore pid: {}\n", 162a17651b1SMemoryShore address.data(), 163a17651b1SMemoryShore error_code, 164a17651b1SMemoryShore crate::process::ProcessManager::current_pid().data() 165a17651b1SMemoryShore ); 166a17651b1SMemoryShore //TODO https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/mm/fault.c#do_kern_addr_fault 167a17651b1SMemoryShore } 168a17651b1SMemoryShore 169a17651b1SMemoryShore /// 用户态缺页异常处理 170a17651b1SMemoryShore /// ## 参数 171a17651b1SMemoryShore /// 172a17651b1SMemoryShore /// - `regs`: 中断栈帧 173a17651b1SMemoryShore /// - `error_code`: 错误标志 174a17651b1SMemoryShore /// - `address`: 发生缺页异常的虚拟地址 do_user_addr_fault( regs: &'static TrapFrame, error_code: X86PfErrorCode, address: VirtAddr, )175a17651b1SMemoryShore pub unsafe fn do_user_addr_fault( 176a17651b1SMemoryShore regs: &'static TrapFrame, 177a17651b1SMemoryShore error_code: X86PfErrorCode, 178a17651b1SMemoryShore address: VirtAddr, 179a17651b1SMemoryShore ) { 180a17651b1SMemoryShore let rflags = RFlags::from_bits_truncate(regs.rflags); 181a17651b1SMemoryShore let mut flags: FaultFlags = FaultFlags::FAULT_FLAG_ALLOW_RETRY 182a17651b1SMemoryShore | FaultFlags::FAULT_FLAG_KILLABLE 183a17651b1SMemoryShore | FaultFlags::FAULT_FLAG_INTERRUPTIBLE; 184a17651b1SMemoryShore 185a17651b1SMemoryShore if error_code & (X86PfErrorCode::X86_PF_USER | X86PfErrorCode::X86_PF_INSTR) 186a17651b1SMemoryShore == X86PfErrorCode::X86_PF_INSTR 187a17651b1SMemoryShore { 188a17651b1SMemoryShore Self::page_fault_oops(regs, error_code, address); 189a17651b1SMemoryShore } 190a17651b1SMemoryShore 191a17651b1SMemoryShore let feature = x86::cpuid::CpuId::new() 192a17651b1SMemoryShore .get_extended_feature_info() 193a17651b1SMemoryShore .unwrap(); 194a17651b1SMemoryShore if unlikely( 195a17651b1SMemoryShore feature.has_smap() 196a17651b1SMemoryShore && !error_code.contains(X86PfErrorCode::X86_PF_USER) 197a17651b1SMemoryShore && rflags.contains(RFlags::FLAGS_AC), 198a17651b1SMemoryShore ) { 199a17651b1SMemoryShore Self::page_fault_oops(regs, error_code, address); 200a17651b1SMemoryShore } 201a17651b1SMemoryShore 202a17651b1SMemoryShore if unlikely(error_code.contains(X86PfErrorCode::X86_PF_RSVD)) { 203a17651b1SMemoryShore // TODO https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/mm/fault.c#pgtable_bad 204a17651b1SMemoryShore panic!( 205a17651b1SMemoryShore "Reserved bits are never expected to be set, error_code: {:#b}, address: {:#x}", 206a17651b1SMemoryShore error_code, 207a17651b1SMemoryShore address.data() 208a17651b1SMemoryShore ); 209a17651b1SMemoryShore } 210a17651b1SMemoryShore 211a17651b1SMemoryShore if regs.is_from_user() { 212a17651b1SMemoryShore unsafe { CurrentIrqArch::interrupt_enable() }; 213a17651b1SMemoryShore flags |= FaultFlags::FAULT_FLAG_USER; 214a17651b1SMemoryShore } else if rflags.contains(RFlags::FLAGS_IF) { 215a17651b1SMemoryShore unsafe { CurrentIrqArch::interrupt_enable() }; 216a17651b1SMemoryShore } 217a17651b1SMemoryShore 218a17651b1SMemoryShore if error_code.contains(X86PfErrorCode::X86_PF_SHSTK) { 219a17651b1SMemoryShore flags |= FaultFlags::FAULT_FLAG_WRITE; 220a17651b1SMemoryShore } 221a17651b1SMemoryShore if error_code.contains(X86PfErrorCode::X86_PF_WRITE) { 222a17651b1SMemoryShore flags |= FaultFlags::FAULT_FLAG_WRITE; 223a17651b1SMemoryShore } 224a17651b1SMemoryShore if error_code.contains(X86PfErrorCode::X86_PF_INSTR) { 225a17651b1SMemoryShore flags |= FaultFlags::FAULT_FLAG_INSTRUCTION; 226a17651b1SMemoryShore } 227a17651b1SMemoryShore 228a17651b1SMemoryShore let current_address_space: Arc<AddressSpace> = AddressSpace::current().unwrap(); 229*cf7f801eSMemoryShore let mut space_guard = current_address_space.write_irqsave(); 230a17651b1SMemoryShore let mut fault; 231a17651b1SMemoryShore loop { 232a17651b1SMemoryShore let vma = space_guard.mappings.find_nearest(address); 233a17651b1SMemoryShore // let vma = space_guard.mappings.contains(address); 234a17651b1SMemoryShore 235*cf7f801eSMemoryShore let vma = match vma { 236*cf7f801eSMemoryShore Some(vma) => vma, 237*cf7f801eSMemoryShore None => { 238*cf7f801eSMemoryShore log::error!( 239a17651b1SMemoryShore "can not find nearest vma, error_code: {:#b}, address: {:#x}", 240a17651b1SMemoryShore error_code, 241a17651b1SMemoryShore address.data(), 242*cf7f801eSMemoryShore ); 243*cf7f801eSMemoryShore let pid = ProcessManager::current_pid(); 244*cf7f801eSMemoryShore let mut info = 245*cf7f801eSMemoryShore SigInfo::new(Signal::SIGSEGV, 0, SigCode::User, SigType::Kill(pid)); 246*cf7f801eSMemoryShore Signal::SIGSEGV 247*cf7f801eSMemoryShore .send_signal_info(Some(&mut info), pid) 248*cf7f801eSMemoryShore .expect("failed to send SIGSEGV to process"); 249*cf7f801eSMemoryShore return; 250*cf7f801eSMemoryShore } 251*cf7f801eSMemoryShore }; 252*cf7f801eSMemoryShore let guard = vma.lock_irqsave(); 253a17651b1SMemoryShore let region = *guard.region(); 254a17651b1SMemoryShore let vm_flags = *guard.vm_flags(); 255a17651b1SMemoryShore drop(guard); 256a17651b1SMemoryShore 257a17651b1SMemoryShore if !region.contains(address) { 258a17651b1SMemoryShore if vm_flags.contains(VmFlags::VM_GROWSDOWN) { 259a17651b1SMemoryShore space_guard 260a17651b1SMemoryShore .extend_stack(region.start() - address) 261a17651b1SMemoryShore .unwrap_or_else(|_| { 262a17651b1SMemoryShore panic!( 263a17651b1SMemoryShore "user stack extend failed, error_code: {:#b}, address: {:#x}", 264a17651b1SMemoryShore error_code, 265a17651b1SMemoryShore address.data(), 266a17651b1SMemoryShore ) 267a17651b1SMemoryShore }); 268a17651b1SMemoryShore } else { 269*cf7f801eSMemoryShore log::error!( 270a17651b1SMemoryShore "No mapped vma, error_code: {:#b}, address: {:#x}", 271a17651b1SMemoryShore error_code, 272a17651b1SMemoryShore address.data(), 273*cf7f801eSMemoryShore ); 274*cf7f801eSMemoryShore let pid = ProcessManager::current_pid(); 275*cf7f801eSMemoryShore let mut info = 276*cf7f801eSMemoryShore SigInfo::new(Signal::SIGSEGV, 0, SigCode::User, SigType::Kill(pid)); 277*cf7f801eSMemoryShore Signal::SIGSEGV 278*cf7f801eSMemoryShore .send_signal_info(Some(&mut info), pid) 279*cf7f801eSMemoryShore .expect("failed to send SIGSEGV to process"); 280*cf7f801eSMemoryShore return; 281a17651b1SMemoryShore } 282a17651b1SMemoryShore } 283a17651b1SMemoryShore 284a17651b1SMemoryShore if unlikely(Self::vma_access_error(vma.clone(), error_code)) { 285a17651b1SMemoryShore panic!( 286a17651b1SMemoryShore "vma access error, error_code: {:#b}, address: {:#x}", 287a17651b1SMemoryShore error_code, 288a17651b1SMemoryShore address.data(), 289a17651b1SMemoryShore ); 290a17651b1SMemoryShore } 291a17651b1SMemoryShore let mapper = &mut space_guard.user_mapper.utable; 292*cf7f801eSMemoryShore let message = PageFaultMessage::new(vma.clone(), address, flags, mapper); 293a17651b1SMemoryShore 294*cf7f801eSMemoryShore fault = PageFaultHandler::handle_mm_fault(message); 295a17651b1SMemoryShore 296a17651b1SMemoryShore if fault.contains(VmFaultReason::VM_FAULT_COMPLETED) { 297a17651b1SMemoryShore return; 298a17651b1SMemoryShore } 299a17651b1SMemoryShore 300a17651b1SMemoryShore if unlikely(fault.contains(VmFaultReason::VM_FAULT_RETRY)) { 301a17651b1SMemoryShore flags |= FaultFlags::FAULT_FLAG_TRIED; 302a17651b1SMemoryShore } else { 303a17651b1SMemoryShore break; 304a17651b1SMemoryShore } 305a17651b1SMemoryShore } 306a17651b1SMemoryShore 307a17651b1SMemoryShore let vm_fault_error = VmFaultReason::VM_FAULT_OOM 308a17651b1SMemoryShore | VmFaultReason::VM_FAULT_SIGBUS 309a17651b1SMemoryShore | VmFaultReason::VM_FAULT_SIGSEGV 310a17651b1SMemoryShore | VmFaultReason::VM_FAULT_HWPOISON 311a17651b1SMemoryShore | VmFaultReason::VM_FAULT_HWPOISON_LARGE 312a17651b1SMemoryShore | VmFaultReason::VM_FAULT_FALLBACK; 313a17651b1SMemoryShore 314a17651b1SMemoryShore if likely(!fault.contains(vm_fault_error)) { 315a17651b1SMemoryShore panic!("fault error: {:?}", fault) 316a17651b1SMemoryShore } 317a17651b1SMemoryShore } 318a17651b1SMemoryShore } 319