1 use riscv::register::satp; 2 use sbi_rt::{HartMask, SbiRet}; 3 use system_error::SystemError; 4 5 use crate::{ 6 arch::MMArch, 7 driver::open_firmware::fdt::open_firmware_fdt_driver, 8 libs::spinlock::SpinLock, 9 mm::{ 10 allocator::{ 11 buddy::BuddyAllocator, 12 page_frame::{FrameAllocator, PageFrameCount, PageFrameUsage, PhysPageFrame}, 13 }, 14 kernel_mapper::KernelMapper, 15 page::{EntryFlags, PageEntry, PAGE_1G_SHIFT}, 16 ucontext::UserMapper, 17 MemoryManagementArch, PageTableKind, PhysAddr, VirtAddr, VmFlags, 18 }, 19 smp::cpu::ProcessorId, 20 }; 21 22 use self::init::{riscv_mm_init, INITIAL_PGTABLE_VALUE}; 23 24 pub mod bump; 25 pub(super) mod init; 26 27 pub type PageMapper = crate::mm::page::PageMapper<RiscV64MMArch, LockedFrameAllocator>; 28 29 /// 内核起始物理地址 30 pub(self) static mut KERNEL_BEGIN_PA: PhysAddr = PhysAddr::new(0); 31 /// 内核结束的物理地址 32 pub(self) static mut KERNEL_END_PA: PhysAddr = PhysAddr::new(0); 33 /// 内核起始虚拟地址 34 pub(self) static mut KERNEL_BEGIN_VA: VirtAddr = VirtAddr::new(0); 35 /// 内核结束虚拟地址 36 pub(self) static mut KERNEL_END_VA: VirtAddr = VirtAddr::new(0); 37 38 pub(self) static INNER_ALLOCATOR: SpinLock<Option<BuddyAllocator<MMArch>>> = SpinLock::new(None); 39 40 /// RiscV64的内存管理架构结构体(sv39) 41 #[derive(Debug, Clone, Copy, Hash)] 42 pub struct RiscV64MMArch; 43 44 impl RiscV64MMArch { 45 /// 使远程cpu的TLB中，指定地址范围的页失效 46 #[allow(dead_code)] 47 pub fn remote_invalidate_page( 48 cpu: ProcessorId, 49 address: VirtAddr, 50 size: usize, 51 ) -> Result<(), SbiRet> { 52 let r = sbi_rt::remote_sfence_vma(Into::into(cpu), address.data(), size); 53 if r.is_ok() { 54 return Ok(()); 55 } else { 56 return Err(r); 57 } 58 } 59 60 /// 使指定远程cpu的TLB中，所有范围的页失效 61 #[allow(dead_code)] 62 pub fn remote_invalidate_all(cpu: ProcessorId) -> Result<(), SbiRet> { 63 let r = Self::remote_invalidate_page( 64 cpu, 65 VirtAddr::new(0), 66 1 << RiscV64MMArch::ENTRY_ADDRESS_SHIFT, 67 ); 68 69 return r; 70 } 71 72 pub fn remote_invalidate_all_with_mask(mask: HartMask) -> Result<(), SbiRet> { 73 let r = sbi_rt::remote_sfence_vma(mask, 0, 1 << RiscV64MMArch::ENTRY_ADDRESS_SHIFT); 74 if r.is_ok() { 75 return Ok(()); 76 } else { 77 return Err(r); 78 } 79 } 80 } 81 82 /// 内核空间起始地址在顶层页表中的索引 83 const KERNEL_TOP_PAGE_ENTRY_NO: usize = (RiscV64MMArch::PHYS_OFFSET 84 & ((1 << RiscV64MMArch::ENTRY_ADDRESS_SHIFT) - 1)) 85 >> (RiscV64MMArch::ENTRY_ADDRESS_SHIFT - RiscV64MMArch::PAGE_ENTRY_SHIFT); 86 87 impl MemoryManagementArch for RiscV64MMArch { 88 /// riscv64暂不支持缺页中断 89 const PAGE_FAULT_ENABLED: bool = false; 90 91 const PAGE_SHIFT: usize = 12; 92 93 const PAGE_ENTRY_SHIFT: usize = 9; 94 95 /// sv39分页只有三级 96 const PAGE_LEVELS: usize = 3; 97 98 const ENTRY_ADDRESS_SHIFT: usize = 39; 99 100 const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT 101 | Self::ENTRY_FLAG_READWRITE 102 | Self::ENTRY_FLAG_DIRTY 103 | Self::ENTRY_FLAG_ACCESSED 104 | Self::ENTRY_FLAG_GLOBAL; 105 106 const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT; 107 108 const ENTRY_FLAG_PRESENT: usize = 1 << 0; 109 110 const ENTRY_FLAG_READONLY: usize = (1 << 1); 111 112 const ENTRY_FLAG_WRITEABLE: usize = (1 << 2); 113 114 const ENTRY_FLAG_READWRITE: usize = (1 << 2) | (1 << 1); 115 116 const ENTRY_FLAG_USER: usize = (1 << 4); 117 const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - (1 << 10); 118 const ENTRY_FLAG_WRITE_THROUGH: usize = (2 << 61); 119 120 const ENTRY_FLAG_CACHE_DISABLE: usize = (2 << 61); 121 122 const ENTRY_FLAG_NO_EXEC: usize = 0; 123 124 const ENTRY_FLAG_EXEC: usize = (1 << 3); 125 const ENTRY_FLAG_ACCESSED: usize = (1 << 6); 126 const ENTRY_FLAG_DIRTY: usize = (1 << 7); 127 const ENTRY_FLAG_GLOBAL: usize = (1 << 5); 128 129 const PHYS_OFFSET: usize = 0xffff_ffc0_0000_0000; 130 const KERNEL_LINK_OFFSET: usize = 0x1000000; 131 132 const USER_END_VADDR: crate::mm::VirtAddr = VirtAddr::new(0x0000_003f_ffff_ffff); 133 134 const USER_BRK_START: crate::mm::VirtAddr = VirtAddr::new(0x0000_001f_ffff_ffff); 135 136 const USER_STACK_START: crate::mm::VirtAddr = VirtAddr::new(0x0000_001f_ffa0_0000); 137 138 /// 在距离sv39的顶端还有64M的位置，设置为FIXMAP的起始地址 139 const FIXMAP_START_VADDR: VirtAddr = VirtAddr::new(0xffff_ffff_fc00_0000); 140 /// 设置1MB的fixmap空间 141 const FIXMAP_SIZE: usize = 256 * 4096; 142 143 /// 在距离sv39的顶端还有2G的位置，设置为MMIO空间的起始地址 144 const MMIO_BASE: VirtAddr = VirtAddr::new(0xffff_ffff_8000_0000); 145 /// 设置1g的MMIO空间 146 const MMIO_SIZE: usize = 1 << PAGE_1G_SHIFT; 147 148 const ENTRY_FLAG_HUGE_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE; 149 150 #[inline(never)] 151 unsafe fn init() { 152 riscv_mm_init().expect("init kernel memory management architecture failed"); 153 } 154 155 unsafe fn arch_post_init() { 156 // 映射fdt 157 open_firmware_fdt_driver() 158 .map_fdt() 159 .expect("openfirmware map fdt failed"); 160 } 161 162 unsafe fn invalidate_page(address: VirtAddr) { 163 riscv::asm::sfence_vma(0, address.data()); 164 } 165 166 unsafe fn invalidate_all() { 167 riscv::asm::sfence_vma_all(); 168 } 169 170 unsafe fn table(_table_kind: PageTableKind) -> PhysAddr { 171 // phys page number 172 let ppn = riscv::register::satp::read().ppn(); 173 174 let paddr = PhysPageFrame::from_ppn(ppn).phys_address(); 175 176 return paddr; 177 } 178 179 unsafe fn set_table(_table_kind: PageTableKind, table: PhysAddr) { 180 let ppn = PhysPageFrame::new(table).ppn(); 181 riscv::asm::sfence_vma_all(); 182 satp::set(satp::Mode::Sv39, 0, ppn); 183 } 184 185 fn virt_is_valid(virt: VirtAddr) -> bool { 186 virt.is_canonical() 187 } 188 189 fn initial_page_table() -> PhysAddr { 190 unsafe { INITIAL_PGTABLE_VALUE } 191 } 192 193 fn setup_new_usermapper() -> Result<UserMapper, SystemError> { 194 let new_umapper: crate::mm::page::PageMapper<MMArch, LockedFrameAllocator> = unsafe { 195 PageMapper::create(PageTableKind::User, LockedFrameAllocator) 196 .ok_or(SystemError::ENOMEM)? 197 }; 198 199 let current_ktable: KernelMapper = KernelMapper::lock(); 200 let copy_mapping = |pml4_entry_no| unsafe { 201 let entry: PageEntry<RiscV64MMArch> = current_ktable 202 .table() 203 .entry(pml4_entry_no) 204 .unwrap_or_else(|| panic!("entry {} not found", pml4_entry_no)); 205 new_umapper.table().set_entry(pml4_entry_no, entry) 206 }; 207 208 // 复制内核的映射 209 for pml4_entry_no in KERNEL_TOP_PAGE_ENTRY_NO..512 { 210 copy_mapping(pml4_entry_no); 211 } 212 213 return Ok(crate::mm::ucontext::UserMapper::new(new_umapper)); 214 } 215 216 unsafe fn phys_2_virt(phys: PhysAddr) -> Option<VirtAddr> { 217 // riscv的内核文件所占用的空间，由于重定位而导致不满足线性偏移量的关系 218 // 因此这里需要特殊处理 219 if phys >= KERNEL_BEGIN_PA && phys < KERNEL_END_PA { 220 let r = KERNEL_BEGIN_VA + (phys - KERNEL_BEGIN_PA); 221 return Some(r); 222 } 223 224 if let Some(vaddr) = phys.data().checked_add(Self::PHYS_OFFSET) { 225 return Some(VirtAddr::new(vaddr)); 226 } else { 227 return None; 228 } 229 } 230 231 unsafe fn virt_2_phys(virt: VirtAddr) -> Option<PhysAddr> { 232 if virt >= KERNEL_BEGIN_VA && virt < KERNEL_END_VA { 233 let r = KERNEL_BEGIN_PA + (virt - KERNEL_BEGIN_VA); 234 return Some(r); 235 } 236 237 if let Some(paddr) = virt.data().checked_sub(Self::PHYS_OFFSET) { 238 let r = PhysAddr::new(paddr); 239 return Some(r); 240 } else { 241 return None; 242 } 243 } 244 245 fn make_entry(paddr: PhysAddr, page_flags: usize) -> usize { 246 let ppn = PhysPageFrame::new(paddr).ppn(); 247 let r = ((ppn & ((1 << 54) - 1)) << 10) | page_flags; 248 return r; 249 } 250 251 fn vma_access_permitted( 252 _vma: alloc::sync::Arc<crate::mm::ucontext::LockedVMA>, 253 _write: bool, 254 _execute: bool, 255 _foreign: bool, 256 ) -> bool { 257 true 258 } 259 260 const PAGE_NONE: usize = Self::ENTRY_FLAG_GLOBAL | Self::ENTRY_FLAG_READONLY; 261 262 const PAGE_READ: usize = PAGE_ENTRY_BASE | Self::ENTRY_FLAG_READONLY; 263 264 const PAGE_WRITE: usize = 265 PAGE_ENTRY_BASE | Self::ENTRY_FLAG_READONLY | Self::ENTRY_FLAG_WRITEABLE; 266 267 const PAGE_EXEC: usize = PAGE_ENTRY_BASE | Self::ENTRY_FLAG_EXEC; 268 269 const PAGE_READ_EXEC: usize = 270 PAGE_ENTRY_BASE | Self::ENTRY_FLAG_READONLY | Self::ENTRY_FLAG_EXEC; 271 272 const PAGE_WRITE_EXEC: usize = PAGE_ENTRY_BASE 273 | Self::ENTRY_FLAG_READONLY 274 | Self::ENTRY_FLAG_EXEC 275 | Self::ENTRY_FLAG_WRITEABLE; 276 277 const PAGE_COPY: usize = Self::PAGE_READ; 278 const PAGE_COPY_EXEC: usize = Self::PAGE_READ_EXEC; 279 const PAGE_SHARED: usize = Self::PAGE_WRITE; 280 const PAGE_SHARED_EXEC: usize = Self::PAGE_WRITE_EXEC; 281 282 const PAGE_COPY_NOEXEC: usize = 0; 283 const PAGE_READONLY: usize = 0; 284 const PAGE_READONLY_EXEC: usize = 0; 285 286 const PROTECTION_MAP: [EntryFlags<MMArch>; 16] = protection_map(); 287 } 288 289 const fn protection_map() -> [EntryFlags<MMArch>; 16] { 290 let mut map = [0; 16]; 291 map[VmFlags::VM_NONE.bits()] = MMArch::PAGE_NONE; 292 map[VmFlags::VM_READ.bits()] = MMArch::PAGE_READONLY; 293 map[VmFlags::VM_WRITE.bits()] = MMArch::PAGE_COPY; 294 map[VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] = MMArch::PAGE_COPY; 295 map[VmFlags::VM_EXEC.bits()] = MMArch::PAGE_READONLY_EXEC; 296 map[VmFlags::VM_EXEC.bits() | VmFlags::VM_READ.bits()] = MMArch::PAGE_READONLY_EXEC; 297 map[VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits()] = MMArch::PAGE_COPY_EXEC; 298 map[VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] = 299 MMArch::PAGE_COPY_EXEC; 300 map[VmFlags::VM_SHARED.bits()] = MMArch::PAGE_NONE; 301 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_READ.bits()] = MMArch::PAGE_READONLY; 302 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_WRITE.bits()] = MMArch::PAGE_SHARED; 303 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] = 304 MMArch::PAGE_SHARED; 305 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits()] = MMArch::PAGE_READONLY_EXEC; 306 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits() | VmFlags::VM_READ.bits()] = 307 MMArch::PAGE_READONLY_EXEC; 308 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits()] = 309 MMArch::PAGE_SHARED_EXEC; 310 map[VmFlags::VM_SHARED.bits() 311 | VmFlags::VM_EXEC.bits() 312 | VmFlags::VM_WRITE.bits() 313 | VmFlags::VM_READ.bits()] = MMArch::PAGE_SHARED_EXEC; 314 let mut ret = [unsafe { EntryFlags::from_data(0) }; 16]; 315 let mut index = 0; 316 while index < 16 { 317 ret[index] = unsafe { EntryFlags::from_data(map[index]) }; 318 index += 1; 319 } 320 ret 321 } 322 323 const PAGE_ENTRY_BASE: usize = RiscV64MMArch::ENTRY_FLAG_PRESENT 324 | RiscV64MMArch::ENTRY_FLAG_ACCESSED 325 | RiscV64MMArch::ENTRY_FLAG_USER; 326 327 impl VirtAddr { 328 /// 判断虚拟地址是否合法 329 #[inline(always)] 330 pub fn is_canonical(self) -> bool { 331 let x = self.data() & RiscV64MMArch::PHYS_OFFSET; 332 // 如果x为0，说明虚拟地址的高位为0，是合法的用户地址 333 // 如果x为PHYS_OFFSET，说明虚拟地址的高位全为1，是合法的内核地址 334 return x == 0 || x == RiscV64MMArch::PHYS_OFFSET; 335 } 336 } 337 338 /// 获取内核地址默认的页面标志 339 pub unsafe fn kernel_page_flags<A: MemoryManagementArch>(_virt: VirtAddr) -> EntryFlags<A> { 340 EntryFlags::from_data(RiscV64MMArch::ENTRY_FLAG_DEFAULT_PAGE) 341 .set_user(false) 342 .set_execute(true) 343 } 344 345 /// 全局的页帧分配器 346 #[derive(Debug, Clone, Copy, Hash)] 347 pub struct LockedFrameAllocator; 348 349 impl FrameAllocator for LockedFrameAllocator { 350 unsafe fn allocate(&mut self, count: PageFrameCount) -> Option<(PhysAddr, PageFrameCount)> { 351 if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() { 352 return allocator.allocate(count); 353 } else { 354 return None; 355 } 356 } 357 358 unsafe fn free(&mut self, address: crate::mm::PhysAddr, count: PageFrameCount) { 359 assert!(count.data().is_power_of_two()); 360 if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() { 361 return allocator.free(address, count); 362 } 363 } 364 365 unsafe fn usage(&self) -> PageFrameUsage { 366 if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() { 367 return allocator.usage(); 368 } else { 369 panic!("usage error"); 370 } 371 } 372 } 373