1 pub mod barrier;
2 pub mod bump;
3 pub mod fault;
4 pub mod pkru;
5
6 use alloc::sync::Arc;
7 use alloc::vec::Vec;
8 use hashbrown::HashSet;
9 use log::{debug, info};
10 use x86::time::rdtsc;
11 use x86_64::registers::model_specific::EferFlags;
12
13 use crate::driver::serial::serial8250::send_to_default_serial8250_port;
14
15 use crate::init::boot::boot_callbacks;
16 use crate::libs::align::page_align_up;
17 use crate::libs::lib_ui::screen_manager::scm_disable_put_to_window;
18 use crate::libs::spinlock::SpinLock;
19
20 use crate::mm::allocator::page_frame::{FrameAllocator, PageFrameCount, PageFrameUsage};
21 use crate::mm::memblock::mem_block_manager;
22 use crate::mm::ucontext::LockedVMA;
23 use crate::{
24 arch::MMArch,
25 mm::allocator::{buddy::BuddyAllocator, bump::BumpAllocator},
26 };
27
28 use crate::mm::kernel_mapper::KernelMapper;
29 use crate::mm::page::{EntryFlags, PageEntry, PAGE_1G_SHIFT};
30 use crate::mm::{MemoryManagementArch, PageTableKind, PhysAddr, VirtAddr, VmFlags};
31
32 use system_error::SystemError;
33
34 use core::arch::asm;
35 use core::fmt::Debug;
36
37 use core::sync::atomic::{compiler_fence, AtomicBool, Ordering};
38
39 use super::kvm::vmx::vmcs::VmcsFields;
40 use super::kvm::vmx::vmx_asm_wrapper::vmx_vmread;
41
42 pub type PageMapper =
43 crate::mm::page::PageMapper<crate::arch::x86_64::mm::X86_64MMArch, LockedFrameAllocator>;
44
45 /// 初始的CR3寄存器的值，用于内存管理初始化时，创建的第一个内核页表的位置
46 static mut INITIAL_CR3_VALUE: PhysAddr = PhysAddr::new(0);
47
48 static INNER_ALLOCATOR: SpinLock<Option<BuddyAllocator<MMArch>>> = SpinLock::new(None);
49
50 #[derive(Clone, Copy, Debug)]
51 pub struct X86_64MMBootstrapInfo {
52 kernel_load_base_paddr: usize,
53 kernel_code_start: usize,
54 kernel_code_end: usize,
55 kernel_data_end: usize,
56 kernel_rodata_end: usize,
57 start_brk: usize,
58 }
59
60 pub(super) static mut BOOTSTRAP_MM_INFO: Option<X86_64MMBootstrapInfo> = None;
61
x86_64_set_kernel_load_base_paddr(paddr: PhysAddr)62 pub(super) fn x86_64_set_kernel_load_base_paddr(paddr: PhysAddr) {
63 unsafe {
64 BOOTSTRAP_MM_INFO.as_mut().unwrap().kernel_load_base_paddr = paddr.data();
65 }
66 }
67
68 /// @brief X86_64的内存管理架构结构体
69 #[derive(Debug, Clone, Copy, Hash)]
70 pub struct X86_64MMArch;
71
72 /// XD标志位是否被保留
73 static XD_RESERVED: AtomicBool = AtomicBool::new(false);
74
75 impl MemoryManagementArch for X86_64MMArch {
76 /// X86目前支持缺页中断
77 const PAGE_FAULT_ENABLED: bool = true;
78 /// 4K页
79 const PAGE_SHIFT: usize = 12;
80
81 /// 每个页表项占8字节，总共有512个页表项
82 const PAGE_ENTRY_SHIFT: usize = 9;
83
84 /// 四级页表（PML4T、PDPT、PDT、PT）
85 const PAGE_LEVELS: usize = 4;
86
87 /// 页表项的有效位的index。在x86_64中，页表项的第[0, 47]位表示地址和flag，
88 /// 第[48, 51]位表示保留。因此，有效位的index为52。
89 /// 请注意，第63位是XD位，表示是否允许执行。
90 const ENTRY_ADDRESS_SHIFT: usize = 52;
91
92 const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT;
93
94 const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT;
95
96 const ENTRY_FLAG_PRESENT: usize = 1 << 0;
97
98 const ENTRY_FLAG_READONLY: usize = 0;
99
100 const ENTRY_FLAG_WRITEABLE: usize = 1 << 1;
101 const ENTRY_FLAG_READWRITE: usize = 1 << 1;
102
103 const ENTRY_FLAG_USER: usize = 1 << 2;
104
105 const ENTRY_FLAG_WRITE_THROUGH: usize = 1 << 3;
106
107 const ENTRY_FLAG_CACHE_DISABLE: usize = 1 << 4;
108
109 const ENTRY_FLAG_NO_EXEC: usize = 1 << 63;
110 /// x86_64不存在EXEC标志位，只有NO_EXEC（XD）标志位
111 const ENTRY_FLAG_EXEC: usize = 0;
112
113 const ENTRY_FLAG_ACCESSED: usize = 1 << 5;
114 const ENTRY_FLAG_DIRTY: usize = 1 << 6;
115 const ENTRY_FLAG_HUGE_PAGE: usize = 1 << 7;
116 const ENTRY_FLAG_GLOBAL: usize = 1 << 8;
117
118 /// 物理地址与虚拟地址的偏移量
119 /// 0xffff_8000_0000_0000
120 const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1);
121 const KERNEL_LINK_OFFSET: usize = 0x100000;
122
123 // 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/arch/x86/include/asm/page_64_types.h#75
124 const USER_END_VADDR: VirtAddr =
125 VirtAddr::new((Self::PAGE_ADDRESS_SIZE >> 1) - Self::PAGE_SIZE);
126 const USER_BRK_START: VirtAddr = VirtAddr::new(0x700000000000);
127 const USER_STACK_START: VirtAddr = VirtAddr::new(0x6ffff0a00000);
128
129 const FIXMAP_START_VADDR: VirtAddr = VirtAddr::new(0xffffb00000000000);
130 /// 设置FIXMAP区域大小为16M
131 const FIXMAP_SIZE: usize = 256 * 4096 * 16;
132
133 const MMIO_BASE: VirtAddr = VirtAddr::new(0xffffa10000000000);
134 const MMIO_SIZE: usize = 1 << PAGE_1G_SHIFT;
135
136 /// @brief 获取物理内存区域
init()137 unsafe fn init() {
138 extern "C" {
139 fn _text();
140 fn _etext();
141 fn _edata();
142 fn _erodata();
143 fn _end();
144 fn _default_kernel_load_base();
145 }
146
147 Self::init_xd_rsvd();
148
149 let bootstrap_info = X86_64MMBootstrapInfo {
150 kernel_load_base_paddr: _default_kernel_load_base as usize,
151 kernel_code_start: _text as usize,
152 kernel_code_end: _etext as usize,
153 kernel_data_end: _edata as usize,
154 kernel_rodata_end: _erodata as usize,
155 start_brk: _end as usize,
156 };
157
158 unsafe {
159 BOOTSTRAP_MM_INFO = Some(bootstrap_info);
160 }
161
162 // 初始化物理内存区域
163 boot_callbacks()
164 .early_init_memory_blocks()
165 .expect("init memory area failed");
166
167 debug!("bootstrap info: {:?}", unsafe { BOOTSTRAP_MM_INFO });
168 debug!("phys[0]=virt[0x{:x}]", unsafe {
169 MMArch::phys_2_virt(PhysAddr::new(0)).unwrap().data()
170 });
171
172 // 初始化内存管理器
173 unsafe { allocator_init() };
174
175 send_to_default_serial8250_port("x86 64 mm init done\n\0".as_bytes());
176 }
177
178 /// @brief 刷新TLB中，关于指定虚拟地址的条目
invalidate_page(address: VirtAddr)179 unsafe fn invalidate_page(address: VirtAddr) {
180 compiler_fence(Ordering::SeqCst);
181 asm!("invlpg [{0}]", in(reg) address.data(), options(nostack, preserves_flags));
182 compiler_fence(Ordering::SeqCst);
183 }
184
185 /// @brief 刷新TLB中，所有的条目
invalidate_all()186 unsafe fn invalidate_all() {
187 compiler_fence(Ordering::SeqCst);
188 // 通过设置cr3寄存器，来刷新整个TLB
189 Self::set_table(PageTableKind::User, Self::table(PageTableKind::User));
190 compiler_fence(Ordering::SeqCst);
191 }
192
193 /// @brief 获取顶级页表的物理地址
table(table_kind: PageTableKind) -> PhysAddr194 unsafe fn table(table_kind: PageTableKind) -> PhysAddr {
195 match table_kind {
196 PageTableKind::Kernel | PageTableKind::User => {
197 compiler_fence(Ordering::SeqCst);
198 let cr3 = x86::controlregs::cr3() as usize;
199 compiler_fence(Ordering::SeqCst);
200 return PhysAddr::new(cr3);
201 }
202 PageTableKind::EPT => {
203 let eptp =
204 vmx_vmread(VmcsFields::CTRL_EPTP_PTR as u32).expect("Failed to read eptp");
205 return PhysAddr::new(eptp as usize);
206 }
207 }
208 }
209
210 /// @brief 设置顶级页表的物理地址到处理器中
set_table(_table_kind: PageTableKind, table: PhysAddr)211 unsafe fn set_table(_table_kind: PageTableKind, table: PhysAddr) {
212 compiler_fence(Ordering::SeqCst);
213 asm!("mov cr3, {}", in(reg) table.data(), options(nostack, preserves_flags));
214 compiler_fence(Ordering::SeqCst);
215 }
216
217 /// @brief 判断虚拟地址是否合法
virt_is_valid(virt: VirtAddr) -> bool218 fn virt_is_valid(virt: VirtAddr) -> bool {
219 return virt.is_canonical();
220 }
221
222 /// 获取内存管理初始化时，创建的第一个内核页表的地址
initial_page_table() -> PhysAddr223 fn initial_page_table() -> PhysAddr {
224 unsafe {
225 return INITIAL_CR3_VALUE;
226 }
227 }
228
229 /// @brief 创建新的顶层页表
230 ///
231 /// 该函数会创建页表并复制内核的映射到新的页表中
232 ///
233 /// @return 新的页表
setup_new_usermapper() -> Result<crate::mm::ucontext::UserMapper, SystemError>234 fn setup_new_usermapper() -> Result<crate::mm::ucontext::UserMapper, SystemError> {
235 let new_umapper: crate::mm::page::PageMapper<X86_64MMArch, LockedFrameAllocator> = unsafe {
236 PageMapper::create(PageTableKind::User, LockedFrameAllocator)
237 .ok_or(SystemError::ENOMEM)?
238 };
239
240 let current_ktable: KernelMapper = KernelMapper::lock();
241 let copy_mapping = |pml4_entry_no| unsafe {
242 let entry: PageEntry<X86_64MMArch> = current_ktable
243 .table()
244 .entry(pml4_entry_no)
245 .unwrap_or_else(|| panic!("entry {} not found", pml4_entry_no));
246 new_umapper.table().set_entry(pml4_entry_no, entry)
247 };
248
249 // 复制内核的映射
250 for pml4_entry_no in MMArch::PAGE_KERNEL_INDEX..MMArch::PAGE_ENTRY_NUM {
251 copy_mapping(pml4_entry_no);
252 }
253
254 return Ok(crate::mm::ucontext::UserMapper::new(new_umapper));
255 }
256
257 const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT;
258
259 const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1;
260
261 const PAGE_MASK: usize = !(Self::PAGE_OFFSET_MASK);
262
263 const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT;
264
265 const PAGE_ADDRESS_SIZE: usize = 1 << Self::PAGE_ADDRESS_SHIFT;
266
267 const PAGE_ADDRESS_MASK: usize = Self::PAGE_ADDRESS_SIZE - Self::PAGE_SIZE;
268
269 const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT);
270
271 const PAGE_ENTRY_NUM: usize = 1 << Self::PAGE_ENTRY_SHIFT;
272
273 const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRY_NUM - 1;
274
275 const PAGE_KERNEL_INDEX: usize = (Self::PHYS_OFFSET & Self::PAGE_ADDRESS_MASK)
276 >> (Self::PAGE_ADDRESS_SHIFT - Self::PAGE_ENTRY_SHIFT);
277
278 const PAGE_NEGATIVE_MASK: usize = !((Self::PAGE_ADDRESS_SIZE) - 1);
279
280 const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_SHIFT;
281
282 const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - Self::PAGE_SIZE;
283
284 const ENTRY_FLAGS_MASK: usize = !Self::ENTRY_ADDRESS_MASK;
285
read<T>(address: VirtAddr) -> T286 unsafe fn read<T>(address: VirtAddr) -> T {
287 return core::ptr::read(address.data() as *const T);
288 }
289
write<T>(address: VirtAddr, value: T)290 unsafe fn write<T>(address: VirtAddr, value: T) {
291 core::ptr::write(address.data() as *mut T, value);
292 }
293
write_bytes(address: VirtAddr, value: u8, count: usize)294 unsafe fn write_bytes(address: VirtAddr, value: u8, count: usize) {
295 core::ptr::write_bytes(address.data() as *mut u8, value, count);
296 }
297
phys_2_virt(phys: PhysAddr) -> Option<VirtAddr>298 unsafe fn phys_2_virt(phys: PhysAddr) -> Option<VirtAddr> {
299 if let Some(vaddr) = phys.data().checked_add(Self::PHYS_OFFSET) {
300 return Some(VirtAddr::new(vaddr));
301 } else {
302 return None;
303 }
304 }
305
virt_2_phys(virt: VirtAddr) -> Option<PhysAddr>306 unsafe fn virt_2_phys(virt: VirtAddr) -> Option<PhysAddr> {
307 if let Some(paddr) = virt.data().checked_sub(Self::PHYS_OFFSET) {
308 return Some(PhysAddr::new(paddr));
309 } else {
310 return None;
311 }
312 }
313
314 #[inline(always)]
make_entry(paddr: PhysAddr, page_flags: usize) -> usize315 fn make_entry(paddr: PhysAddr, page_flags: usize) -> usize {
316 return paddr.data() | page_flags;
317 }
318
vma_access_permitted( vma: Arc<LockedVMA>, write: bool, execute: bool, foreign: bool, ) -> bool319 fn vma_access_permitted(
320 vma: Arc<LockedVMA>,
321 write: bool,
322 execute: bool,
323 foreign: bool,
324 ) -> bool {
325 if execute {
326 return true;
327 }
328 if foreign | vma.is_foreign() {
329 return true;
330 }
331 pkru::pkru_allows_pkey(pkru::vma_pkey(vma), write)
332 }
333
334 const PROTECTION_MAP: [EntryFlags<MMArch>; 16] = protection_map();
335
336 const PAGE_NONE: usize =
337 Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_ACCESSED | Self::ENTRY_FLAG_GLOBAL;
338
339 const PAGE_SHARED: usize = Self::ENTRY_FLAG_PRESENT
340 | Self::ENTRY_FLAG_READWRITE
341 | Self::ENTRY_FLAG_USER
342 | Self::ENTRY_FLAG_ACCESSED
343 | Self::ENTRY_FLAG_NO_EXEC;
344
345 const PAGE_SHARED_EXEC: usize = Self::ENTRY_FLAG_PRESENT
346 | Self::ENTRY_FLAG_READWRITE
347 | Self::ENTRY_FLAG_USER
348 | Self::ENTRY_FLAG_ACCESSED;
349
350 const PAGE_COPY_NOEXEC: usize = Self::ENTRY_FLAG_PRESENT
351 | Self::ENTRY_FLAG_USER
352 | Self::ENTRY_FLAG_ACCESSED
353 | Self::ENTRY_FLAG_NO_EXEC;
354
355 const PAGE_COPY_EXEC: usize =
356 Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_USER | Self::ENTRY_FLAG_ACCESSED;
357
358 const PAGE_COPY: usize = Self::ENTRY_FLAG_PRESENT
359 | Self::ENTRY_FLAG_USER
360 | Self::ENTRY_FLAG_ACCESSED
361 | Self::ENTRY_FLAG_NO_EXEC;
362
363 const PAGE_READONLY: usize = Self::ENTRY_FLAG_PRESENT
364 | Self::ENTRY_FLAG_USER
365 | Self::ENTRY_FLAG_ACCESSED
366 | Self::ENTRY_FLAG_NO_EXEC;
367
368 const PAGE_READONLY_EXEC: usize =
369 Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_USER | Self::ENTRY_FLAG_ACCESSED;
370
371 const PAGE_READ: usize = 0;
372 const PAGE_READ_EXEC: usize = 0;
373 const PAGE_WRITE: usize = 0;
374 const PAGE_WRITE_EXEC: usize = 0;
375 const PAGE_EXEC: usize = 0;
376 }
377
378 /// 获取保护标志的映射表
379 ///
380 ///
381 /// ## 返回值
382 /// - `[usize; 16]`: 长度为16的映射表
protection_map() -> [EntryFlags<MMArch>; 16]383 const fn protection_map() -> [EntryFlags<MMArch>; 16] {
384 let mut map = [unsafe { EntryFlags::from_data(0) }; 16];
385 unsafe {
386 map[VmFlags::VM_NONE.bits()] = EntryFlags::from_data(MMArch::PAGE_NONE);
387 map[VmFlags::VM_READ.bits()] = EntryFlags::from_data(MMArch::PAGE_READONLY);
388 map[VmFlags::VM_WRITE.bits()] = EntryFlags::from_data(MMArch::PAGE_COPY);
389 map[VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] =
390 EntryFlags::from_data(MMArch::PAGE_COPY);
391 map[VmFlags::VM_EXEC.bits()] = EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
392 map[VmFlags::VM_EXEC.bits() | VmFlags::VM_READ.bits()] =
393 EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
394 map[VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits()] =
395 EntryFlags::from_data(MMArch::PAGE_COPY_EXEC);
396 map[VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] =
397 EntryFlags::from_data(MMArch::PAGE_COPY_EXEC);
398 map[VmFlags::VM_SHARED.bits()] = EntryFlags::from_data(MMArch::PAGE_NONE);
399 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_READ.bits()] =
400 EntryFlags::from_data(MMArch::PAGE_READONLY);
401 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_WRITE.bits()] =
402 EntryFlags::from_data(MMArch::PAGE_SHARED);
403 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_WRITE.bits() | VmFlags::VM_READ.bits()] =
404 EntryFlags::from_data(MMArch::PAGE_SHARED);
405 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits()] =
406 EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
407 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits() | VmFlags::VM_READ.bits()] =
408 EntryFlags::from_data(MMArch::PAGE_READONLY_EXEC);
409 map[VmFlags::VM_SHARED.bits() | VmFlags::VM_EXEC.bits() | VmFlags::VM_WRITE.bits()] =
410 EntryFlags::from_data(MMArch::PAGE_SHARED_EXEC);
411 map[VmFlags::VM_SHARED.bits()
412 | VmFlags::VM_EXEC.bits()
413 | VmFlags::VM_WRITE.bits()
414 | VmFlags::VM_READ.bits()] = EntryFlags::from_data(MMArch::PAGE_SHARED_EXEC);
415 }
416 // if X86_64MMArch::is_xd_reserved() {
417 // map.iter_mut().for_each(|x| *x &= !Self::ENTRY_FLAG_NO_EXEC)
418 // }
419 map
420 }
421
422 impl X86_64MMArch {
init_xd_rsvd()423 fn init_xd_rsvd() {
424 // 读取ia32-EFER寄存器的值
425 let efer: EferFlags = x86_64::registers::model_specific::Efer::read();
426 if !efer.contains(EferFlags::NO_EXECUTE_ENABLE) {
427 // NO_EXECUTE_ENABLE是false，那么就设置xd_reserved为true
428 debug!("NO_EXECUTE_ENABLE is false, set XD_RESERVED to true");
429 XD_RESERVED.store(true, Ordering::Relaxed);
430 }
431 compiler_fence(Ordering::SeqCst);
432 }
433
434 /// 判断XD标志位是否被保留
is_xd_reserved() -> bool435 pub fn is_xd_reserved() -> bool {
436 // return XD_RESERVED.load(Ordering::Relaxed);
437
438 // 由于暂时不支持execute disable，因此直接返回true
439 // 不支持的原因是，目前好像没有能正确的设置page-level的xd位，会触发page fault
440 return true;
441 }
442 }
443
444 impl VirtAddr {
445 /// @brief 判断虚拟地址是否合法
446 #[inline(always)]
is_canonical(self) -> bool447 pub fn is_canonical(self) -> bool {
448 let x = self.data() & X86_64MMArch::PHYS_OFFSET;
449 // 如果x为0，说明虚拟地址的高位为0，是合法的用户地址
450 // 如果x为PHYS_OFFSET，说明虚拟地址的高位全为1，是合法的内核地址
451 return x == 0 || x == X86_64MMArch::PHYS_OFFSET;
452 }
453 }
454
allocator_init()455 unsafe fn allocator_init() {
456 let virt_offset = VirtAddr::new(page_align_up(BOOTSTRAP_MM_INFO.unwrap().start_brk));
457
458 let phy_offset = unsafe { MMArch::virt_2_phys(virt_offset) }.unwrap();
459
460 mem_block_manager()
461 .reserve_block(PhysAddr::new(0), phy_offset.data())
462 .expect("Failed to reserve block");
463 let mut bump_allocator = BumpAllocator::<X86_64MMArch>::new(phy_offset.data());
464 debug!(
465 "BumpAllocator created, offset={:?}",
466 bump_allocator.offset()
467 );
468
469 // 暂存初始在head.S中指定的页表的地址，后面再考虑是否需要把它加到buddy的可用空间里面！
470 // 现在不加的原因是，我担心会有安全漏洞问题：这些初始的页表，位于内核的数据段。如果归还到buddy，
471 // 可能会产生一定的安全风险（有的代码可能根据虚拟地址来进行安全校验）
472 let _old_page_table = MMArch::table(PageTableKind::Kernel);
473
474 let new_page_table: PhysAddr;
475 // 使用bump分配器，把所有的内存页都映射到页表
476 {
477 // 用bump allocator创建新的页表
478 let mut mapper: crate::mm::page::PageMapper<MMArch, &mut BumpAllocator<MMArch>> =
479 crate::mm::page::PageMapper::<MMArch, _>::create(
480 PageTableKind::Kernel,
481 &mut bump_allocator,
482 )
483 .expect("Failed to create page mapper");
484 new_page_table = mapper.table().phys();
485 debug!("PageMapper created");
486
487 // 取消最开始时候，在head.S中指定的映射(暂时不刷新TLB)
488 {
489 let table = mapper.table();
490 let empty_entry = PageEntry::<MMArch>::from_usize(0);
491 for i in 0..MMArch::PAGE_ENTRY_NUM {
492 table
493 .set_entry(i, empty_entry)
494 .expect("Failed to empty page table entry");
495 }
496 }
497 debug!("Successfully emptied page table");
498
499 let total_num = mem_block_manager().total_initial_memory_regions();
500 for i in 0..total_num {
501 let area = mem_block_manager().get_initial_memory_region(i).unwrap();
502 // debug!("area: base={:?}, size={:#x}, end={:?}", area.base, area.size, area.base + area.size);
503 for i in 0..area.size.div_ceil(MMArch::PAGE_SIZE) {
504 let paddr = area.base.add(i * MMArch::PAGE_SIZE);
505 let vaddr = unsafe { MMArch::phys_2_virt(paddr) }.unwrap();
506 let flags = kernel_page_flags::<MMArch>(vaddr);
507
508 let flusher = mapper
509 .map_phys(vaddr, paddr, flags)
510 .expect("Failed to map frame");
511 // 暂时不刷新TLB
512 flusher.ignore();
513 }
514 }
515 }
516
517 unsafe {
518 INITIAL_CR3_VALUE = new_page_table;
519 }
520 debug!(
521 "After mapping all physical memory, DragonOS used: {} KB",
522 bump_allocator.offset() / 1024
523 );
524
525 // 初始化buddy_allocator
526 let buddy_allocator = unsafe { BuddyAllocator::<X86_64MMArch>::new(bump_allocator).unwrap() };
527 // 设置全局的页帧分配器
528 unsafe { set_inner_allocator(buddy_allocator) };
529 info!("Successfully initialized buddy allocator");
530 // 关闭显示输出
531 scm_disable_put_to_window();
532
533 // make the new page table current
534 {
535 let mut binding = INNER_ALLOCATOR.lock();
536 let mut allocator_guard = binding.as_mut().unwrap();
537 debug!("To enable new page table.");
538 compiler_fence(Ordering::SeqCst);
539 let mapper = crate::mm::page::PageMapper::<MMArch, _>::new(
540 PageTableKind::Kernel,
541 new_page_table,
542 &mut allocator_guard,
543 );
544 compiler_fence(Ordering::SeqCst);
545 mapper.make_current();
546 compiler_fence(Ordering::SeqCst);
547 debug!("New page table enabled");
548 }
549 debug!("Successfully enabled new page table");
550 }
551
552 #[no_mangle]
rs_test_buddy()553 pub extern "C" fn rs_test_buddy() {
554 test_buddy();
555 }
test_buddy()556 pub fn test_buddy() {
557 // 申请内存然后写入数据然后free掉
558 // 总共申请200MB内存
559 const TOTAL_SIZE: usize = 200 * 1024 * 1024;
560
561 for i in 0..10 {
562 debug!("Test buddy, round: {i}");
563 // 存放申请的内存块
564 let mut v: Vec<(PhysAddr, PageFrameCount)> = Vec::with_capacity(60 * 1024);
565 // 存放已经申请的内存块的地址（用于检查重复）
566 let mut addr_set: HashSet<PhysAddr> = HashSet::new();
567
568 let mut allocated = 0usize;
569
570 let mut free_count = 0usize;
571
572 while allocated < TOTAL_SIZE {
573 let mut random_size = 0u64;
574 unsafe { x86::random::rdrand64(&mut random_size) };
575 // 一次最多申请4M
576 random_size %= 1024 * 4096;
577 if random_size == 0 {
578 continue;
579 }
580 let random_size =
581 core::cmp::min(page_align_up(random_size as usize), TOTAL_SIZE - allocated);
582 let random_size = PageFrameCount::from_bytes(random_size.next_power_of_two()).unwrap();
583 // 获取帧
584 let (paddr, allocated_frame_count) =
585 unsafe { LockedFrameAllocator.allocate(random_size).unwrap() };
586 assert!(allocated_frame_count.data().is_power_of_two());
587 assert!(paddr.data() % MMArch::PAGE_SIZE == 0);
588 unsafe {
589 assert!(MMArch::phys_2_virt(paddr)
590 .as_ref()
591 .unwrap()
592 .check_aligned(allocated_frame_count.data() * MMArch::PAGE_SIZE));
593 }
594 allocated += allocated_frame_count.data() * MMArch::PAGE_SIZE;
595 v.push((paddr, allocated_frame_count));
596 assert!(addr_set.insert(paddr), "duplicate address: {:?}", paddr);
597
598 // 写入数据
599 let vaddr = unsafe { MMArch::phys_2_virt(paddr).unwrap() };
600 let slice = unsafe {
601 core::slice::from_raw_parts_mut(
602 vaddr.data() as *mut u8,
603 allocated_frame_count.data() * MMArch::PAGE_SIZE,
604 )
605 };
606 for (i, item) in slice.iter_mut().enumerate() {
607 *item = ((i + unsafe { rdtsc() } as usize) % 256) as u8;
608 }
609
610 // 随机释放一个内存块
611 if !v.is_empty() {
612 let mut random_index = 0u64;
613 unsafe { x86::random::rdrand64(&mut random_index) };
614 // 70%概率释放
615 if random_index % 10 > 7 {
616 continue;
617 }
618 random_index %= v.len() as u64;
619 let random_index = random_index as usize;
620 let (paddr, allocated_frame_count) = v.remove(random_index);
621 assert!(addr_set.remove(&paddr));
622 unsafe { LockedFrameAllocator.free(paddr, allocated_frame_count) };
623 free_count += allocated_frame_count.data() * MMArch::PAGE_SIZE;
624 }
625 }
626
627 debug!(
628 "Allocated {} MB memory, release: {} MB, no release: {} bytes",
629 allocated / 1024 / 1024,
630 free_count / 1024 / 1024,
631 (allocated - free_count)
632 );
633
634 debug!("Now, to release buddy memory");
635 // 释放所有的内存
636 for (paddr, allocated_frame_count) in v {
637 unsafe { LockedFrameAllocator.free(paddr, allocated_frame_count) };
638 assert!(addr_set.remove(&paddr));
639 free_count += allocated_frame_count.data() * MMArch::PAGE_SIZE;
640 }
641
642 debug!("release done!, allocated: {allocated}, free_count: {free_count}");
643 }
644 }
645
646 /// 全局的页帧分配器
647 #[derive(Debug, Clone, Copy, Hash)]
648 pub struct LockedFrameAllocator;
649
650 impl FrameAllocator for LockedFrameAllocator {
allocate(&mut self, mut count: PageFrameCount) -> Option<(PhysAddr, PageFrameCount)>651 unsafe fn allocate(&mut self, mut count: PageFrameCount) -> Option<(PhysAddr, PageFrameCount)> {
652 count = count.next_power_of_two();
653 if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() {
654 return allocator.allocate(count);
655 } else {
656 return None;
657 }
658 }
659
free(&mut self, address: crate::mm::PhysAddr, count: PageFrameCount)660 unsafe fn free(&mut self, address: crate::mm::PhysAddr, count: PageFrameCount) {
661 assert!(count.data().is_power_of_two());
662 if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() {
663 return allocator.free(address, count);
664 }
665 }
666
usage(&self) -> PageFrameUsage667 unsafe fn usage(&self) -> PageFrameUsage {
668 if let Some(ref mut allocator) = *INNER_ALLOCATOR.lock_irqsave() {
669 return allocator.usage();
670 } else {
671 panic!("usage error");
672 }
673 }
674 }
675
676 /// 获取内核地址默认的页面标志
kernel_page_flags<A: MemoryManagementArch>(virt: VirtAddr) -> EntryFlags<A>677 pub unsafe fn kernel_page_flags<A: MemoryManagementArch>(virt: VirtAddr) -> EntryFlags<A> {
678 let info: X86_64MMBootstrapInfo = BOOTSTRAP_MM_INFO.unwrap();
679
680 if virt.data() >= info.kernel_code_start && virt.data() < info.kernel_code_end {
681 // Remap kernel code execute
682 return EntryFlags::new().set_execute(true).set_write(true);
683 } else if virt.data() >= info.kernel_data_end && virt.data() < info.kernel_rodata_end {
684 // Remap kernel rodata read only
685 return EntryFlags::new().set_execute(true);
686 } else {
687 return EntryFlags::new().set_write(true).set_execute(true);
688 }
689 }
690
set_inner_allocator(allocator: BuddyAllocator<MMArch>)691 unsafe fn set_inner_allocator(allocator: BuddyAllocator<MMArch>) {
692 static FLAG: AtomicBool = AtomicBool::new(false);
693 if FLAG
694 .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
695 .is_err()
696 {
697 panic!("Cannot set inner allocator twice!");
698 }
699 *INNER_ALLOCATOR.lock() = Some(allocator);
700 }
701
702 /// 低地址重映射的管理器
703 ///
704 /// 低地址重映射的管理器，在smp初始化完成之前，需要使用低地址的映射，因此需要在smp初始化完成之后，取消这一段映射
705 pub struct LowAddressRemapping;
706
707 impl LowAddressRemapping {
708 // 映射64M
709 const REMAP_SIZE: usize = 64 * 1024 * 1024;
710
remap_at_low_address(mapper: &mut PageMapper)711 pub unsafe fn remap_at_low_address(mapper: &mut PageMapper) {
712 for i in 0..(Self::REMAP_SIZE / MMArch::PAGE_SIZE) {
713 let paddr = PhysAddr::new(i * MMArch::PAGE_SIZE);
714 let vaddr = VirtAddr::new(i * MMArch::PAGE_SIZE);
715 let flags = kernel_page_flags::<MMArch>(vaddr);
716
717 let flusher = mapper
718 .map_phys(vaddr, paddr, flags)
719 .expect("Failed to map frame");
720 // 暂时不刷新TLB
721 flusher.ignore();
722 }
723 }
724
725 /// 取消低地址的映射
unmap_at_low_address(mapper: &mut PageMapper, flush: bool)726 pub unsafe fn unmap_at_low_address(mapper: &mut PageMapper, flush: bool) {
727 for i in 0..(Self::REMAP_SIZE / MMArch::PAGE_SIZE) {
728 let vaddr = VirtAddr::new(i * MMArch::PAGE_SIZE);
729 let (_, _, flusher) = mapper
730 .unmap_phys(vaddr, true)
731 .expect("Failed to unmap frame");
732 if !flush {
733 flusher.ignore();
734 }
735 }
736 }
737 }
738