1 use crate::libs::spinlock::{SpinLock, SpinLockGuard}; 2 use crate::{ 3 arch::asm::current::current_pcb, 4 include::bindings::bindings::{ 5 initial_mm, mm_create_vma, mm_unmap, vm_area_del, vm_area_free, vm_area_struct, vm_flags_t, 6 vma_find, EINVAL, ENOMEM, EPERM, MMIO_BASE, MMIO_TOP, PAGE_1G_SHIFT, PAGE_1G_SIZE, 7 PAGE_2M_SIZE, PAGE_4K_SHIFT, PAGE_4K_SIZE, VM_DONTCOPY, VM_IO, 8 }, 9 kdebug, kerror, 10 }; 11 use alloc::{boxed::Box, collections::LinkedList, vec::Vec}; 12 use core::{mem, ptr::null_mut}; 13 14 // 最大的伙伴块的幂 15 const MMIO_BUDDY_MAX_EXP: u32 = PAGE_1G_SHIFT; 16 // 最小的伙伴块的幂 17 const MMIO_BUDDY_MIN_EXP: u32 = PAGE_4K_SHIFT; 18 // 内存池数组的范围 19 const MMIO_BUDDY_REGION_COUNT: u32 = MMIO_BUDDY_MAX_EXP - MMIO_BUDDY_MIN_EXP + 1; 20 21 lazy_static! { 22 pub static ref MMIO_POOL: MmioBuddyMemPool = MmioBuddyMemPool::new(); 23 } 24 25 pub enum MmioResult { 26 SUCCESS, 27 EINVAL, 28 ENOFOUND, 29 WRONGEXP, 30 ISEMPTY, 31 } 32 33 /// @brief buddy内存池 34 pub struct MmioBuddyMemPool { 35 pool_start_addr: u64, 36 pool_size: u64, 37 free_regions: [SpinLock<MmioFreeRegionList>; MMIO_BUDDY_REGION_COUNT as usize], 38 } 39 impl Default for MmioBuddyMemPool { 40 fn default() -> Self { 41 MmioBuddyMemPool { 42 pool_start_addr: MMIO_BASE as u64, 43 pool_size: (MMIO_TOP - MMIO_BASE) as u64, 44 free_regions: unsafe { mem::zeroed() }, 45 } 46 } 47 } 48 impl MmioBuddyMemPool { 49 fn new() -> Self { 50 return MmioBuddyMemPool { 51 ..Default::default() 52 }; 53 } 54 55 /// @brief 创建新的地址区域结构体 56 /// 57 /// @param vaddr 虚拟地址 58 /// 59 /// @return 创建好的地址区域结构体 60 fn create_region(&self, vaddr: u64) -> Box<MmioBuddyAddrRegion> { 61 let mut region: Box<MmioBuddyAddrRegion> = Box::new(MmioBuddyAddrRegion::new()); 62 region.vaddr = vaddr; 63 return region; 64 } 65 66 /// @brief 将内存块归还给buddy 67 /// 68 /// @param vaddr 虚拟地址 69 /// 70 /// @param exp 内存空间的大小（2^exp） 71 /// 72 /// @param list_guard 【exp】对应的链表 73 /// 74 /// @return Ok(i32) 返回0 75 /// 76 /// @return Err(i32) 返回错误码 77 fn give_back_block(&self, vaddr: u64, exp: u32) -> Result<i32, i32> { 78 // 确保内存对齐，低位都要为0 79 if (vaddr & ((1 << exp) - 1)) != 0 { 80 return Err(-(EINVAL as i32)); 81 } 82 let region: Box<MmioBuddyAddrRegion> = self.create_region(vaddr); 83 // 加入buddy 84 let list_guard: &mut SpinLockGuard<MmioFreeRegionList> = 85 &mut self.free_regions[exp2index(exp)].lock(); 86 self.push_block(region, list_guard); 87 return Ok(0); 88 } 89 90 /// @brief 将给定大小为2^{exp}的内存块一分为二，并插入内存块大小为2^{exp-1}的链表中 91 /// 92 /// @param region 要被分割的地址区域结构体（保证其已经从链表中取出） 93 /// 94 /// @param exp 要被分割的地址区域的大小的幂 95 /// 96 /// @param list_guard 【exp-1】对应的链表 97 fn split_block( 98 &self, 99 region: Box<MmioBuddyAddrRegion>, 100 exp: u32, 101 low_list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 102 ) { 103 let vaddr: u64 = self.calculate_block_vaddr(region.vaddr, exp - 1); 104 let new_region: Box<MmioBuddyAddrRegion> = self.create_region(vaddr); 105 self.push_block(region, low_list_guard); 106 self.push_block(new_region, low_list_guard); 107 } 108 109 /// @brief 从buddy中申请一块指定大小的内存区域 110 /// 111 /// @param exp 要申请的内存块的大小的幂(2^exp) 112 /// 113 /// @param list_guard exp对应的链表 114 /// 115 /// @return Ok(Box<MmioBuddyAddrRegion>) 符合要求的内存区域。 116 /// 117 /// @return Err(MmioResult) 118 /// - 没有满足要求的内存块时，返回ENOFOUND 119 /// - 申请的内存块大小超过合法范围，返回WRONGEXP 120 /// - 调用函数出错时，返回出错函数对应错误码 121 fn query_addr_region( 122 &self, 123 exp: u32, 124 list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 125 ) -> Result<Box<MmioBuddyAddrRegion>, MmioResult> { 126 // 申请范围错误 127 if exp < MMIO_BUDDY_MIN_EXP || exp > MMIO_BUDDY_MAX_EXP { 128 kdebug!("query_addr_region: exp wrong"); 129 return Err(MmioResult::WRONGEXP); 130 } 131 // 没有恰好符合要求的内存块 132 // 注意：exp对应的链表list_guard已上锁【注意避免死锁问题】 133 if list_guard.num_free == 0 { 134 // 找到最小符合申请范围的内存块 135 // 将大的内存块依次分成小块内存，直到能够满足exp大小，即将exp+1分成两块exp 136 for e in exp + 1..MMIO_BUDDY_MAX_EXP + 1 { 137 if self.free_regions[exp2index(e) as usize].lock().num_free == 0 { 138 continue; 139 } 140 for e2 in (exp + 1..e + 1).rev() { 141 match self.pop_block(&mut self.free_regions[exp2index(e2) as usize].lock()) { 142 Ok(region) => { 143 if e2 != exp + 1 { 144 // 要将分裂后的内存块插入到更小的链表中 145 let low_list_guard: &mut SpinLockGuard<MmioFreeRegionList> = 146 &mut self.free_regions[exp2index(e2 - 1) as usize].lock(); 147 self.split_block(region, e2, low_list_guard); 148 } else { 149 // 由于exp对应的链表list_guard已经被锁住了 不能再加锁 150 // 所以直接将list_guard传入 151 self.split_block(region, e2, list_guard); 152 } 153 } 154 Err(err) => { 155 kdebug!("buddy_pop_region get wrong"); 156 return Err(err); 157 } 158 } 159 } 160 break; 161 } 162 // 判断是否获得了exp大小的内存块 163 if list_guard.num_free > 0 { 164 return Ok(list_guard.list.pop_back().unwrap()); 165 } 166 // 拆分大内存块无法获得exp大小内存块 167 // 尝试用小内存块合成 168 // 即将两块exp合成一块exp+1 169 for e in MMIO_BUDDY_MIN_EXP..exp { 170 if e != exp - 1 { 171 let high_list_guard: &mut SpinLockGuard<MmioFreeRegionList> = 172 &mut self.free_regions[exp2index(exp + 1)].lock(); 173 match self.merge_all_exp( 174 e, 175 &mut self.free_regions[exp2index(e) as usize].lock(), 176 high_list_guard, 177 ) { 178 Ok(_) => continue, 179 Err(err) => { 180 return Err(err); 181 } 182 } 183 } else { 184 match self.merge_all_exp( 185 e, 186 &mut self.free_regions[exp2index(e) as usize].lock(), 187 list_guard, 188 ) { 189 Ok(_) => continue, 190 Err(err) => { 191 return Err(err); 192 } 193 } 194 } 195 } 196 197 //判断是否获得了exp大小的内存块 198 if list_guard.num_free > 0 { 199 return Ok(list_guard.list.pop_back().unwrap()); 200 } 201 return Err(MmioResult::ENOFOUND); 202 } else { 203 return Ok(list_guard.list.pop_back().unwrap()); 204 } 205 } 206 207 /// @brief 对query_addr_region进行封装 208 /// 209 /// @param exp 内存区域的大小(2^exp) 210 /// 211 /// @return Ok(Box<MmioBuddyAddrRegion>)符合要求的内存块信息结构体。 212 /// @return Err(MmioResult) 没有满足要求的内存块时，返回__query_addr_region的错误码。 213 fn mmio_buddy_query_addr_region( 214 &self, 215 exp: u32, 216 ) -> Result<Box<MmioBuddyAddrRegion>, MmioResult> { 217 let list_guard: &mut SpinLockGuard<MmioFreeRegionList> = 218 &mut self.free_regions[exp2index(exp)].lock(); 219 match self.query_addr_region(exp, list_guard) { 220 Ok(ret) => return Ok(ret), 221 Err(err) => { 222 kdebug!("mmio_buddy_query_addr_region failed"); 223 return Err(err); 224 } 225 } 226 } 227 /// @brief 往指定的地址空间链表中添加一个地址区域 228 /// 229 /// @param region 要被添加的地址结构体 230 /// 231 /// @param list_guard 目标链表 232 fn push_block( 233 &self, 234 region: Box<MmioBuddyAddrRegion>, 235 list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 236 ) { 237 list_guard.list.push_back(region); 238 list_guard.num_free += 1; 239 } 240 241 /// @brief 根据地址和内存块大小，计算伙伴块虚拟内存的地址 242 #[inline(always)] 243 fn calculate_block_vaddr(&self, vaddr: u64, exp: u32) -> u64 { 244 return vaddr ^ (1 << exp); 245 } 246 247 /// @brief 寻找并弹出指定内存块的伙伴块 248 /// 249 /// @param region 对应内存块的信息 250 /// 251 /// @param exp 内存块大小 252 /// 253 /// @param list_guard 【exp】对应的链表 254 /// 255 /// @return Ok(Box<MmioBuddyAddrRegion) 返回伙伴块的引用 256 /// @return Err(MmioResult) 257 /// - 当链表为空，返回ISEMPTY 258 /// - 没有找到伙伴块，返回ENOFOUND 259 fn pop_buddy_block( 260 &self, 261 vaddr: u64, 262 exp: u32, 263 list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 264 ) -> Result<Box<MmioBuddyAddrRegion>, MmioResult> { 265 if list_guard.list.len() == 0 { 266 return Err(MmioResult::ISEMPTY); 267 } else { 268 //计算伙伴块的地址 269 let buddy_vaddr = self.calculate_block_vaddr(vaddr, exp); 270 271 // element 只会有一个元素 272 let mut element: Vec<Box<MmioBuddyAddrRegion>> = list_guard 273 .list 274 .drain_filter(|x| x.vaddr == buddy_vaddr) 275 .collect(); 276 if element.len() == 1 { 277 list_guard.num_free -= 1; 278 return Ok(element.pop().unwrap()); 279 } 280 281 //没有找到对应的伙伴块 282 return Err(MmioResult::ENOFOUND); 283 } 284 } 285 286 /// @brief 从指定空闲链表中取出内存区域 287 /// 288 /// @param list_guard 【exp】对应的链表 289 /// 290 /// @return Ok(Box<MmioBuddyAddrRegion>) 内存块信息结构体的引用。 291 /// 292 /// @return Err(MmioResult) 当链表为空，无法删除时，返回ISEMPTY 293 fn pop_block( 294 &self, 295 list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 296 ) -> Result<Box<MmioBuddyAddrRegion>, MmioResult> { 297 if !list_guard.list.is_empty() { 298 list_guard.num_free -= 1; 299 return Ok(list_guard.list.pop_back().unwrap()); 300 } 301 return Err(MmioResult::ISEMPTY); 302 } 303 304 /// @brief 合并所有2^{exp}大小的内存块 305 /// 306 /// @param exp 内存块大小的幂(2^exp) 307 /// 308 /// @param list_guard exp对应的链表 309 /// 310 /// @param high_list_guard exp+1对应的链表 311 /// 312 /// @return Ok(MmioResult) 合并成功返回SUCCESS 313 /// @return Err(MmioResult) 314 /// - 内存块过少，无法合并，返回EINVAL 315 /// - pop_buddy_block调用出错，返回其错误码 316 /// - merge_blocks调用出错，返回其错误码 317 fn merge_all_exp( 318 &self, 319 exp: u32, 320 list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 321 high_list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 322 ) -> Result<MmioResult, MmioResult> { 323 // 至少要两个内存块才能合并 324 if list_guard.num_free <= 1 { 325 return Err(MmioResult::EINVAL); 326 } 327 loop { 328 if list_guard.num_free <= 1 { 329 break; 330 } 331 // 获取内存块 332 let vaddr: u64 = list_guard.list.back().unwrap().vaddr; 333 // 获取伙伴内存块 334 match self.pop_buddy_block(vaddr, exp, list_guard) { 335 Err(err) => { 336 return Err(err); 337 } 338 Ok(buddy_region) => { 339 let region: Box<MmioBuddyAddrRegion> = list_guard.list.pop_back().unwrap(); 340 let copy_region: Box<MmioBuddyAddrRegion> = Box::new(MmioBuddyAddrRegion { 341 vaddr: region.vaddr, 342 }); 343 // 在两块内存都被取出之后才进行合并 344 match self.merge_blocks(region, buddy_region, exp, high_list_guard) { 345 Err(err) => { 346 // 如果合并失败了要将取出来的元素放回去 347 self.push_block(copy_region, list_guard); 348 kdebug!("merge_all_exp: merge_blocks failed"); 349 return Err(err); 350 } 351 Ok(_) => continue, 352 } 353 } 354 } 355 } 356 return Ok(MmioResult::SUCCESS); 357 } 358 359 /// @brief 合并两个【已经从链表中取出】的内存块 360 /// 361 /// @param region_1 第一个内存块 362 /// 363 /// @param region_2 第二个内存 364 /// 365 /// @return Ok(MmioResult) 成功返回SUCCESS 366 /// 367 /// @return Err(MmioResult) 两个内存块不是伙伴块,返回EINVAL 368 fn merge_blocks( 369 &self, 370 region_1: Box<MmioBuddyAddrRegion>, 371 region_2: Box<MmioBuddyAddrRegion>, 372 exp: u32, 373 high_list_guard: &mut SpinLockGuard<MmioFreeRegionList>, 374 ) -> Result<MmioResult, MmioResult> { 375 // 判断是否为伙伴块 376 if region_1.vaddr != self.calculate_block_vaddr(region_2.vaddr, exp) { 377 return Err(MmioResult::EINVAL); 378 } 379 // 将大的块放进下一级链表 380 self.push_block(region_1, high_list_guard); 381 return Ok(MmioResult::SUCCESS); 382 } 383 384 /// @brief 创建一块mmio区域，并将vma绑定到initial_mm 385 /// 386 /// @param size mmio区域的大小（字节） 387 /// 388 /// @param vm_flags 要把vma设置成的标志 389 /// 390 /// @param res_vaddr 返回值-分配得到的虚拟地址 391 /// 392 /// @param res_length 返回值-分配的虚拟地址空间长度 393 /// 394 /// @return Ok(i32) 成功返回0 395 /// 396 /// @return Err(i32) 失败返回错误码 397 fn create_mmio( 398 &self, 399 size: u32, 400 vm_flags: vm_flags_t, 401 res_vaddr: *mut u64, 402 res_length: *mut u64, 403 ) -> Result<i32, i32> { 404 if size > PAGE_1G_SIZE || size == 0 { 405 return Err(-(EPERM as i32)); 406 } 407 let mut retval: i32 = 0; 408 // 计算前导0 409 let mut size_exp: u32 = 31 - size.leading_zeros(); 410 // 记录最终申请的空间大小 411 let mut new_size: u32 = size; 412 // 对齐要申请的空间大小 413 // 如果要申请的空间大小小于4k，则分配4k 414 if size_exp < PAGE_4K_SHIFT { 415 new_size = PAGE_4K_SIZE; 416 size_exp = PAGE_4K_SHIFT; 417 } else if (new_size & (!(1 << size_exp))) != 0 { 418 // 向左对齐空间大小 419 size_exp += 1; 420 new_size = 1 << size_exp; 421 } 422 match MMIO_POOL.mmio_buddy_query_addr_region(size_exp) { 423 Ok(region) => { 424 unsafe { 425 *res_vaddr = region.vaddr; 426 *res_length = new_size as u64; 427 } 428 // 创建vma 429 let flags: u64 = vm_flags | (VM_IO | VM_DONTCOPY) as u64; 430 let len_4k: u64 = (new_size % PAGE_2M_SIZE) as u64; 431 let len_2m: u64 = new_size as u64 - len_4k; 432 let mut loop_i: u64 = 0; 433 // 先分配2M的vma 434 loop { 435 if loop_i >= len_2m { 436 break; 437 } 438 let vma: *mut *mut vm_area_struct = null_mut(); 439 retval = unsafe { 440 mm_create_vma( 441 &mut initial_mm, 442 region.vaddr + loop_i, 443 PAGE_2M_SIZE.into(), 444 flags, 445 null_mut(), 446 vma, 447 ) 448 }; 449 if retval != 0 { 450 kdebug!( 451 "failed to create mmio 2m vma. pid = {:?}", 452 current_pcb().pid 453 ); 454 unsafe { 455 vm_area_del(*vma); 456 vm_area_free(*vma); 457 } 458 return Err(retval); 459 } 460 loop_i += PAGE_2M_SIZE as u64; 461 } 462 // 分配4K的vma 463 loop_i = len_2m; 464 loop { 465 if loop_i >= size as u64 { 466 break; 467 } 468 let vma: *mut *mut vm_area_struct = null_mut(); 469 retval = unsafe { 470 mm_create_vma( 471 &mut initial_mm, 472 region.vaddr + loop_i, 473 PAGE_4K_SIZE.into(), 474 flags, 475 null_mut(), 476 vma, 477 ) 478 }; 479 if retval != 0 { 480 kdebug!( 481 "failed to create mmio 4k vma. pid = {:?}", 482 current_pcb().pid 483 ); 484 unsafe { 485 vm_area_del(*vma); 486 vm_area_free(*vma); 487 } 488 return Err(retval); 489 } 490 loop_i += PAGE_4K_SIZE as u64; 491 } 492 } 493 Err(_) => { 494 kdebug!("failed to create mmio vma.pid = {:?}", current_pcb().pid); 495 return Err(-(ENOMEM as i32)); 496 } 497 } 498 return Ok(retval); 499 } 500 501 /// @brief 取消mmio的映射并将地址空间归还到buddy中 502 /// 503 /// @param vaddr 起始的虚拟地址 504 /// 505 /// @param length 要归还的地址空间的长度 506 /// 507 /// @return Ok(i32) 成功返回0 508 /// 509 /// @return Err(i32) 失败返回错误码 510 fn release_mmio(&self, vaddr: u64, length: u64) -> Result<i32, i32> { 511 //先将要释放的空间取消映射 512 unsafe { 513 mm_unmap(&mut initial_mm, vaddr, length, false); 514 } 515 let mut loop_i: u64 = 0; 516 loop { 517 if loop_i >= length { 518 break; 519 } 520 // 获取要释放的vma的结构体 521 let vma: *mut vm_area_struct = unsafe { vma_find(&mut initial_mm, vaddr + loop_i) }; 522 if vma == null_mut() { 523 kdebug!( 524 "mmio_release failed: vma not found. At address: {:?}, pid = {:?}", 525 vaddr + loop_i, 526 current_pcb().pid 527 ); 528 return Err(-(EINVAL as i32)); 529 } 530 // 检查vma起始地址是否正确 531 if unsafe { (*vma).vm_start != (vaddr + loop_i) } { 532 kdebug!( 533 "mmio_release failed: addr_start is not equal to current: {:?}. pid = {:?}", 534 vaddr + loop_i, 535 current_pcb().pid 536 ); 537 return Err(-(EINVAL as i32)); 538 } 539 // 将vma对应空间归还 540 match MMIO_POOL.give_back_block(unsafe { (*vma).vm_start }, unsafe { 541 31 - ((*vma).vm_end - (*vma).vm_start).leading_zeros() 542 }) { 543 Ok(_) => { 544 loop_i += unsafe { (*vma).vm_end - (*vma).vm_start }; 545 unsafe { 546 vm_area_del(vma); 547 vm_area_free(vma); 548 } 549 } 550 Err(err) => { 551 // vma对应空间没有成功归还的话，就不删除vma 552 kdebug!( 553 "mmio_release give_back failed: pid = {:?}", 554 current_pcb().pid 555 ); 556 return Err(err); 557 } 558 } 559 } 560 return Ok(0); 561 } 562 } 563 564 /// @brief mmio伙伴系统内部的地址区域结构体 565 pub struct MmioBuddyAddrRegion { 566 vaddr: u64, 567 } 568 impl MmioBuddyAddrRegion { 569 pub fn new() -> Self { 570 return MmioBuddyAddrRegion { 571 ..Default::default() 572 }; 573 } 574 } 575 impl Default for MmioBuddyAddrRegion { 576 fn default() -> Self { 577 MmioBuddyAddrRegion { 578 vaddr: Default::default(), 579 } 580 } 581 } 582 583 /// @brief 空闲页数组结构体 584 pub struct MmioFreeRegionList { 585 /// 存储mmio_buddy的地址链表 586 list: LinkedList<Box<MmioBuddyAddrRegion>>, 587 /// 空闲块的数量 588 num_free: i64, 589 } 590 impl MmioFreeRegionList { 591 fn new() -> Self { 592 return MmioFreeRegionList { 593 ..Default::default() 594 }; 595 } 596 } 597 impl Default for MmioFreeRegionList { 598 fn default() -> Self { 599 MmioFreeRegionList { 600 list: Default::default(), 601 num_free: 0, 602 } 603 } 604 } 605 606 /// @brief 初始化mmio的伙伴系统 607 #[no_mangle] 608 pub extern "C" fn __mmio_buddy_init() { 609 // 创建一堆1GB的地址块 610 let cnt_1g_blocks: u32 = ((MMIO_TOP - MMIO_BASE) / PAGE_1G_SIZE as i64) as u32; 611 let mut vaddr_base: u64 = MMIO_BASE as u64; 612 for _ in 0..cnt_1g_blocks { 613 match MMIO_POOL.give_back_block(vaddr_base, PAGE_1G_SHIFT) { 614 Ok(_) => { 615 vaddr_base += PAGE_1G_SIZE as u64; 616 } 617 Err(_) => { 618 kerror!("__mmio_buddy_init failed"); 619 return; 620 } 621 } 622 } 623 } 624 625 /// @brief 将内存对象大小的幂转换成内存池中的数组的下标 626 /// 627 /// @param exp内存大小 628 /// 629 /// @return 内存池数组下标 630 #[inline(always)] 631 fn exp2index(exp: u32) -> usize { 632 return (exp - 12) as usize; 633 } 634 635 /// @brief 创建一块mmio区域，并将vma绑定到initial_mm 636 /// 637 /// @param size mmio区域的大小（字节） 638 /// 639 /// @param vm_flags 要把vma设置成的标志 640 /// 641 /// @param res_vaddr 返回值-分配得到的虚拟地址 642 /// 643 /// @param res_length 返回值-分配的虚拟地址空间长度 644 /// 645 /// @return int 错误码 646 #[no_mangle] 647 pub extern "C" fn mmio_create( 648 size: u32, 649 vm_flags: vm_flags_t, 650 res_vaddr: *mut u64, 651 res_length: *mut u64, 652 ) -> i32 { 653 if let Err(err) = MMIO_POOL.create_mmio(size, vm_flags, res_vaddr, res_length) { 654 return err; 655 } else { 656 return 0; 657 } 658 } 659 660 /// @brief 取消mmio的映射并将地址空间归还到buddy中 661 /// 662 /// @param vaddr 起始的虚拟地址 663 /// 664 /// @param length 要归还的地址空间的长度 665 /// 666 /// @return Ok(i32) 成功返回0 667 /// 668 /// @return Err(i32) 失败返回错误码 669 #[no_mangle] 670 pub extern "C" fn mmio_release(vaddr: u64, length: u64) -> i32 { 671 if let Err(err) = MMIO_POOL.release_mmio(vaddr, length) { 672 return err; 673 } else { 674 return 0; 675 } 676 } 677