1*70f159a3SLoGin use acpi::address; 27a29d4fcSLoGin use riscv::register::satp; 3*70f159a3SLoGin use sbi_rt::{HartMask, SbiRet}; 491e9d4abSLoGin use system_error::SystemError; 591e9d4abSLoGin 6453452ccSLoGin use crate::{ 7453452ccSLoGin arch::MMArch, 8*70f159a3SLoGin kdebug, 9453452ccSLoGin libs::spinlock::SpinLock, 10453452ccSLoGin mm::{ 11453452ccSLoGin allocator::{ 12453452ccSLoGin buddy::BuddyAllocator, 13453452ccSLoGin page_frame::{FrameAllocator, PageFrameCount, PageFrameUsage, PhysPageFrame}, 14453452ccSLoGin }, 15*70f159a3SLoGin kernel_mapper::KernelMapper, 16*70f159a3SLoGin page::{PageEntry, PageFlags}, 17*70f159a3SLoGin ucontext::UserMapper, 187a29d4fcSLoGin MemoryManagementArch, PageTableKind, PhysAddr, VirtAddr, 19453452ccSLoGin }, 20*70f159a3SLoGin smp::cpu::ProcessorId, 214fda81ceSLoGin }; 224fda81ceSLoGin 23453452ccSLoGin use self::init::riscv_mm_init; 24453452ccSLoGin 254fda81ceSLoGin pub mod bump; 26666cffedSLoGin pub(super) mod init; 274fda81ceSLoGin 284fda81ceSLoGin pub type PageMapper = crate::mm::page::PageMapper<RiscV64MMArch, LockedFrameAllocator>; 294fda81ceSLoGin 307a29d4fcSLoGin /// 内核起始物理地址 317a29d4fcSLoGin pub(self) static mut KERNEL_BEGIN_PA: PhysAddr = PhysAddr::new(0); 327a29d4fcSLoGin /// 内核结束的物理地址 337a29d4fcSLoGin pub(self) static mut KERNEL_END_PA: PhysAddr = PhysAddr::new(0); 347a29d4fcSLoGin /// 内核起始虚拟地址 357a29d4fcSLoGin pub(self) static mut KERNEL_BEGIN_VA: VirtAddr = VirtAddr::new(0); 367a29d4fcSLoGin /// 内核结束虚拟地址 377a29d4fcSLoGin pub(self) static mut KERNEL_END_VA: VirtAddr = VirtAddr::new(0); 387a29d4fcSLoGin 39453452ccSLoGin pub(self) static INNER_ALLOCATOR: SpinLock<Option<BuddyAllocator<MMArch>>> = SpinLock::new(None); 40453452ccSLoGin 4174ffde66SLoGin /// RiscV64的内存管理架构结构体(sv39) 424fda81ceSLoGin #[derive(Debug, Clone, Copy, Hash)] 434fda81ceSLoGin pub struct RiscV64MMArch; 444fda81ceSLoGin 4592849878SLoGin impl RiscV64MMArch { 4692849878SLoGin pub const ENTRY_FLAG_GLOBAL: usize = 1 << 5; 47*70f159a3SLoGin 48*70f159a3SLoGin /// 使远程cpu的TLB中,指定地址范围的页失效 49*70f159a3SLoGin pub fn remote_invalidate_page( 50*70f159a3SLoGin cpu: ProcessorId, 51*70f159a3SLoGin address: VirtAddr, 52*70f159a3SLoGin size: usize, 53*70f159a3SLoGin ) -> Result<(), SbiRet> { 54*70f159a3SLoGin let r = sbi_rt::remote_sfence_vma(Into::into(cpu), address.data(), size); 55*70f159a3SLoGin if r.is_ok() { 56*70f159a3SLoGin return Ok(()); 57*70f159a3SLoGin } else { 58*70f159a3SLoGin return Err(r); 5992849878SLoGin } 60*70f159a3SLoGin } 61*70f159a3SLoGin 62*70f159a3SLoGin /// 使指定远程cpu的TLB中,所有范围的页失效 63*70f159a3SLoGin pub fn remote_invalidate_all(cpu: ProcessorId) -> Result<(), SbiRet> { 64*70f159a3SLoGin let r = Self::remote_invalidate_page( 65*70f159a3SLoGin cpu, 66*70f159a3SLoGin VirtAddr::new(0), 67*70f159a3SLoGin 1 << RiscV64MMArch::ENTRY_ADDRESS_SHIFT, 68*70f159a3SLoGin ); 69*70f159a3SLoGin 70*70f159a3SLoGin return r; 71*70f159a3SLoGin } 72*70f159a3SLoGin 73*70f159a3SLoGin pub fn remote_invalidate_all_with_mask(mask: HartMask) -> Result<(), SbiRet> { 74*70f159a3SLoGin let r = sbi_rt::remote_sfence_vma(mask, 0, 1 << RiscV64MMArch::ENTRY_ADDRESS_SHIFT); 75*70f159a3SLoGin if r.is_ok() { 76*70f159a3SLoGin return Ok(()); 77*70f159a3SLoGin } else { 78*70f159a3SLoGin return Err(r); 79*70f159a3SLoGin } 80*70f159a3SLoGin } 81*70f159a3SLoGin } 82*70f159a3SLoGin 83*70f159a3SLoGin /// 内核空间起始地址在顶层页表中的索引 84*70f159a3SLoGin const KERNEL_TOP_PAGE_ENTRY_NO: usize = (RiscV64MMArch::PHYS_OFFSET 85*70f159a3SLoGin & ((1 << RiscV64MMArch::ENTRY_ADDRESS_SHIFT) - 1)) 86*70f159a3SLoGin >> (RiscV64MMArch::ENTRY_ADDRESS_SHIFT - RiscV64MMArch::PAGE_ENTRY_SHIFT); 87*70f159a3SLoGin 884fda81ceSLoGin impl MemoryManagementArch for RiscV64MMArch { 894fda81ceSLoGin const PAGE_SHIFT: usize = 12; 904fda81ceSLoGin 914fda81ceSLoGin const PAGE_ENTRY_SHIFT: usize = 9; 924fda81ceSLoGin 934fda81ceSLoGin /// sv39分页只有三级 944fda81ceSLoGin const PAGE_LEVELS: usize = 3; 954fda81ceSLoGin 964fda81ceSLoGin const ENTRY_ADDRESS_SHIFT: usize = 39; 974fda81ceSLoGin 9892849878SLoGin const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT 9992849878SLoGin | Self::ENTRY_FLAG_READWRITE 10092849878SLoGin | Self::ENTRY_FLAG_DIRTY 10192849878SLoGin | Self::ENTRY_FLAG_ACCESSED 10292849878SLoGin | Self::ENTRY_FLAG_GLOBAL; 1034fda81ceSLoGin 1044fda81ceSLoGin const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; 1054fda81ceSLoGin 1064fda81ceSLoGin const ENTRY_FLAG_PRESENT: usize = 1 << 0; 1074fda81ceSLoGin 10892849878SLoGin const ENTRY_FLAG_READONLY: usize = 0; 1094fda81ceSLoGin 1104fda81ceSLoGin const ENTRY_FLAG_READWRITE: usize = (1 << 2) | (1 << 1); 1114fda81ceSLoGin 1124fda81ceSLoGin const ENTRY_FLAG_USER: usize = (1 << 4); 1134fda81ceSLoGin 1144fda81ceSLoGin const ENTRY_FLAG_WRITE_THROUGH: usize = (2 << 61); 1154fda81ceSLoGin 1164fda81ceSLoGin const ENTRY_FLAG_CACHE_DISABLE: usize = (2 << 61); 1174fda81ceSLoGin 1184fda81ceSLoGin const ENTRY_FLAG_NO_EXEC: usize = 0; 1194fda81ceSLoGin 1204fda81ceSLoGin const ENTRY_FLAG_EXEC: usize = (1 << 3); 12192849878SLoGin const ENTRY_FLAG_ACCESSED: usize = (1 << 6); 12292849878SLoGin const ENTRY_FLAG_DIRTY: usize = (1 << 7); 1234fda81ceSLoGin 1244fda81ceSLoGin const PHYS_OFFSET: usize = 0xffff_ffc0_0000_0000; 125453452ccSLoGin const KERNEL_LINK_OFFSET: usize = 0x1000000; 1264fda81ceSLoGin 1274fda81ceSLoGin const USER_END_VADDR: crate::mm::VirtAddr = VirtAddr::new(0x0000_003f_ffff_ffff); 1284fda81ceSLoGin 1294fda81ceSLoGin const USER_BRK_START: crate::mm::VirtAddr = VirtAddr::new(0x0000_001f_ffff_ffff); 1304fda81ceSLoGin 1314fda81ceSLoGin const USER_STACK_START: crate::mm::VirtAddr = VirtAddr::new(0x0000_001f_ffa0_0000); 1324fda81ceSLoGin 13374ffde66SLoGin /// 在距离sv39的顶端还有1G的位置,设置为FIXMAP的起始地址 13474ffde66SLoGin const FIXMAP_START_VADDR: VirtAddr = VirtAddr::new(0xffff_ffff_8000_0000); 13574ffde66SLoGin /// 设置1MB的fixmap空间 13674ffde66SLoGin const FIXMAP_SIZE: usize = 256 * 4096; 13774ffde66SLoGin 138453452ccSLoGin #[inline(never)] 13945626c85SLoGin unsafe fn init() { 140453452ccSLoGin riscv_mm_init().expect("init kernel memory management architecture failed"); 1414fda81ceSLoGin } 1424fda81ceSLoGin 1437a29d4fcSLoGin unsafe fn invalidate_page(address: VirtAddr) { 1447a29d4fcSLoGin riscv::asm::sfence_vma(0, address.data()); 1454fda81ceSLoGin } 1464fda81ceSLoGin 1474fda81ceSLoGin unsafe fn invalidate_all() { 1487a29d4fcSLoGin riscv::asm::sfence_vma_all(); 1494fda81ceSLoGin } 1504fda81ceSLoGin 1517a29d4fcSLoGin unsafe fn table(_table_kind: PageTableKind) -> PhysAddr { 1527a29d4fcSLoGin // phys page number 1537a29d4fcSLoGin let ppn = riscv::register::satp::read().ppn(); 1547a29d4fcSLoGin 1557a29d4fcSLoGin let paddr = PhysPageFrame::from_ppn(ppn).phys_address(); 1567a29d4fcSLoGin 1577a29d4fcSLoGin return paddr; 1584fda81ceSLoGin } 1594fda81ceSLoGin 1607a29d4fcSLoGin unsafe fn set_table(_table_kind: PageTableKind, table: PhysAddr) { 1617a29d4fcSLoGin let ppn = PhysPageFrame::new(table).ppn(); 1627a29d4fcSLoGin riscv::asm::sfence_vma_all(); 1637a29d4fcSLoGin satp::set(satp::Mode::Sv39, 0, ppn); 1644fda81ceSLoGin } 1654fda81ceSLoGin 166*70f159a3SLoGin fn virt_is_valid(virt: VirtAddr) -> bool { 1677a29d4fcSLoGin virt.is_canonical() 1684fda81ceSLoGin } 1694fda81ceSLoGin 170*70f159a3SLoGin fn initial_page_table() -> PhysAddr { 1714fda81ceSLoGin todo!() 1724fda81ceSLoGin } 1734fda81ceSLoGin 174*70f159a3SLoGin fn setup_new_usermapper() -> Result<UserMapper, SystemError> { 175*70f159a3SLoGin let new_umapper: crate::mm::page::PageMapper<MMArch, LockedFrameAllocator> = unsafe { 176*70f159a3SLoGin PageMapper::create(PageTableKind::User, LockedFrameAllocator) 177*70f159a3SLoGin .ok_or(SystemError::ENOMEM)? 178*70f159a3SLoGin }; 179*70f159a3SLoGin 180*70f159a3SLoGin let current_ktable: KernelMapper = KernelMapper::lock(); 181*70f159a3SLoGin let copy_mapping = |pml4_entry_no| unsafe { 182*70f159a3SLoGin let entry: PageEntry<RiscV64MMArch> = current_ktable 183*70f159a3SLoGin .table() 184*70f159a3SLoGin .entry(pml4_entry_no) 185*70f159a3SLoGin .unwrap_or_else(|| panic!("entry {} not found", pml4_entry_no)); 186*70f159a3SLoGin new_umapper.table().set_entry(pml4_entry_no, entry) 187*70f159a3SLoGin }; 188*70f159a3SLoGin 189*70f159a3SLoGin // 复制内核的映射 190*70f159a3SLoGin for pml4_entry_no in KERNEL_TOP_PAGE_ENTRY_NO..512 { 191*70f159a3SLoGin copy_mapping(pml4_entry_no); 192*70f159a3SLoGin } 193*70f159a3SLoGin 194*70f159a3SLoGin return Ok(crate::mm::ucontext::UserMapper::new(new_umapper)); 1954fda81ceSLoGin } 1967a29d4fcSLoGin 1977a29d4fcSLoGin unsafe fn phys_2_virt(phys: PhysAddr) -> Option<VirtAddr> { 1987a29d4fcSLoGin // riscv的内核文件所占用的空间,由于重定位而导致不满足线性偏移量的关系 1997a29d4fcSLoGin // 因此这里需要特殊处理 2007a29d4fcSLoGin if phys >= KERNEL_BEGIN_PA && phys < KERNEL_END_PA { 2017a29d4fcSLoGin let r = KERNEL_BEGIN_VA + (phys - KERNEL_BEGIN_PA); 2027a29d4fcSLoGin return Some(r); 2037a29d4fcSLoGin } 2047a29d4fcSLoGin 2057a29d4fcSLoGin if let Some(vaddr) = phys.data().checked_add(Self::PHYS_OFFSET) { 2067a29d4fcSLoGin return Some(VirtAddr::new(vaddr)); 2077a29d4fcSLoGin } else { 2087a29d4fcSLoGin return None; 2097a29d4fcSLoGin } 2107a29d4fcSLoGin } 2117a29d4fcSLoGin 2127a29d4fcSLoGin unsafe fn virt_2_phys(virt: VirtAddr) -> Option<PhysAddr> { 2137a29d4fcSLoGin if virt >= KERNEL_BEGIN_VA && virt < KERNEL_END_VA { 2147a29d4fcSLoGin let r = KERNEL_BEGIN_PA + (virt - KERNEL_BEGIN_VA); 2157a29d4fcSLoGin return Some(r); 2167a29d4fcSLoGin } 2177a29d4fcSLoGin 2187a29d4fcSLoGin if let Some(paddr) = virt.data().checked_sub(Self::PHYS_OFFSET) { 2197a29d4fcSLoGin let r = PhysAddr::new(paddr); 2207a29d4fcSLoGin return Some(r); 2217a29d4fcSLoGin } else { 2227a29d4fcSLoGin return None; 2237a29d4fcSLoGin } 2247a29d4fcSLoGin } 2257a29d4fcSLoGin 2267a29d4fcSLoGin fn make_entry(paddr: PhysAddr, page_flags: usize) -> usize { 2277a29d4fcSLoGin let ppn = PhysPageFrame::new(paddr).ppn(); 228453452ccSLoGin let r = ((ppn & ((1 << 54) - 1)) << 10) | page_flags; 2297a29d4fcSLoGin return r; 2307a29d4fcSLoGin } 2317a29d4fcSLoGin } 2327a29d4fcSLoGin 2337a29d4fcSLoGin impl VirtAddr { 2347a29d4fcSLoGin /// 判断虚拟地址是否合法 2357a29d4fcSLoGin #[inline(always)] 2367a29d4fcSLoGin pub fn is_canonical(self) -> bool { 2377a29d4fcSLoGin let x = self.data() & RiscV64MMArch::PHYS_OFFSET; 2387a29d4fcSLoGin // 如果x为0,说明虚拟地址的高位为0,是合法的用户地址 2397a29d4fcSLoGin // 如果x为PHYS_OFFSET,说明虚拟地址的高位全为1,是合法的内核地址 2407a29d4fcSLoGin return x == 0 || x == RiscV64MMArch::PHYS_OFFSET; 2417a29d4fcSLoGin } 2424fda81ceSLoGin } 2434fda81ceSLoGin 2444fda81ceSLoGin /// 获取内核地址默认的页面标志 245453452ccSLoGin pub unsafe fn kernel_page_flags<A: MemoryManagementArch>(_virt: VirtAddr) -> PageFlags<A> { 246453452ccSLoGin PageFlags::from_data(RiscV64MMArch::ENTRY_FLAG_DEFAULT_PAGE) 247453452ccSLoGin .set_user(false) 248453452ccSLoGin .set_execute(true) 2494fda81ceSLoGin } 2504fda81ceSLoGin 2514fda81ceSLoGin /// 全局的页帧分配器 2524fda81ceSLoGin #[derive(Debug, Clone, Copy, Hash)] 2534fda81ceSLoGin pub struct LockedFrameAllocator; 2544fda81ceSLoGin 2554fda81ceSLoGin impl FrameAllocator for LockedFrameAllocator { 2564fda81ceSLoGin unsafe fn allocate(&mut self, count: PageFrameCount) -> Option<(PhysAddr, PageFrameCount)> { 257453452ccSLoGin if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() { 258453452ccSLoGin return allocator.allocate(count); 259453452ccSLoGin } else { 260453452ccSLoGin return None; 261453452ccSLoGin } 2624fda81ceSLoGin } 2634fda81ceSLoGin 2644fda81ceSLoGin unsafe fn free(&mut self, address: crate::mm::PhysAddr, count: PageFrameCount) { 2654fda81ceSLoGin assert!(count.data().is_power_of_two()); 266453452ccSLoGin if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() { 267453452ccSLoGin return allocator.free(address, count); 268453452ccSLoGin } 2694fda81ceSLoGin } 2704fda81ceSLoGin 2714fda81ceSLoGin unsafe fn usage(&self) -> PageFrameUsage { 272453452ccSLoGin if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() { 273453452ccSLoGin return allocator.usage(); 274453452ccSLoGin } else { 275453452ccSLoGin panic!("usage error"); 276453452ccSLoGin } 2774fda81ceSLoGin } 2784fda81ceSLoGin } 279