1 use core::{ 2 hash::Hash, 3 hint::spin_loop, 4 intrinsics::{likely, unlikely}, 5 mem::ManuallyDrop, 6 sync::atomic::{compiler_fence, AtomicBool, AtomicIsize, AtomicUsize, Ordering}, 7 }; 8 9 use alloc::{ 10 string::{String, ToString}, 11 sync::{Arc, Weak}, 12 vec::Vec, 13 }; 14 use hashbrown::HashMap; 15 use system_error::SystemError; 16 17 use crate::{ 18 arch::{ 19 ipc::signal::{AtomicSignal, SigSet, Signal}, 20 process::ArchPCBInfo, 21 sched::sched, 22 CurrentIrqArch, 23 }, 24 driver::tty::tty_core::TtyCore, 25 exception::InterruptArch, 26 filesystem::{ 27 procfs::procfs_unregister_pid, 28 vfs::{file::FileDescriptorVec, FileType}, 29 }, 30 ipc::signal_types::{SigInfo, SigPending, SignalStruct}, 31 kdebug, kinfo, 32 libs::{ 33 align::AlignedBox, 34 casting::DowncastArc, 35 futex::{ 36 constant::{FutexFlag, FUTEX_BITSET_MATCH_ANY}, 37 futex::Futex, 38 }, 39 lock_free_flags::LockFreeFlags, 40 rwlock::{RwLock, RwLockReadGuard, RwLockUpgradableGuard, RwLockWriteGuard}, 41 spinlock::{SpinLock, SpinLockGuard}, 42 wait_queue::WaitQueue, 43 }, 44 mm::{percpu::PerCpuVar, set_IDLE_PROCESS_ADDRESS_SPACE, ucontext::AddressSpace, VirtAddr}, 45 net::socket::SocketInode, 46 sched::{ 47 completion::Completion, 48 core::{sched_enqueue, CPU_EXECUTING}, 49 SchedPolicy, SchedPriority, 50 }, 51 smp::{ 52 cpu::{AtomicProcessorId, ProcessorId}, 53 kick_cpu, 54 }, 55 syscall::{user_access::clear_user, Syscall}, 56 }; 57 58 use self::kthread::WorkerPrivate; 59 60 pub mod abi; 61 pub mod c_adapter; 62 pub mod exec; 63 pub mod exit; 64 pub mod fork; 65 pub mod idle; 66 pub mod kthread; 67 pub mod pid; 68 pub mod process; 69 pub mod resource; 70 pub mod syscall; 71 72 /// 系统中所有进程的pcb 73 static ALL_PROCESS: SpinLock<Option<HashMap<Pid, Arc<ProcessControlBlock>>>> = SpinLock::new(None); 74 75 pub static mut SWITCH_RESULT: Option<PerCpuVar<SwitchResult>> = None; 76 77 /// 一个只改变1次的全局变量，标志进程管理器是否已经初始化完成 78 static mut __PROCESS_MANAGEMENT_INIT_DONE: bool = false; 79 80 #[derive(Debug)] 81 pub struct SwitchResult { 82 pub prev_pcb: Option<Arc<ProcessControlBlock>>, 83 pub next_pcb: Option<Arc<ProcessControlBlock>>, 84 } 85 86 impl SwitchResult { 87 pub fn new() -> Self { 88 Self { 89 prev_pcb: None, 90 next_pcb: None, 91 } 92 } 93 } 94 95 #[derive(Debug)] 96 pub struct ProcessManager; 97 impl ProcessManager { 98 #[inline(never)] 99 fn init() { 100 static INIT_FLAG: AtomicBool = AtomicBool::new(false); 101 if INIT_FLAG 102 .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) 103 .is_err() 104 { 105 panic!("ProcessManager has been initialized!"); 106 } 107 108 unsafe { 109 compiler_fence(Ordering::SeqCst); 110 kdebug!("To create address space for INIT process."); 111 // test_buddy(); 112 set_IDLE_PROCESS_ADDRESS_SPACE( 113 AddressSpace::new(true).expect("Failed to create address space for INIT process."), 114 ); 115 kdebug!("INIT process address space created."); 116 compiler_fence(Ordering::SeqCst); 117 }; 118 119 ALL_PROCESS.lock_irqsave().replace(HashMap::new()); 120 Self::arch_init(); 121 kdebug!("process arch init done."); 122 Self::init_idle(); 123 kdebug!("process idle init done."); 124 125 unsafe { __PROCESS_MANAGEMENT_INIT_DONE = true }; 126 kinfo!("Process Manager initialized."); 127 } 128 129 /// 判断进程管理器是否已经初始化完成 130 pub fn initialized() -> bool { 131 unsafe { __PROCESS_MANAGEMENT_INIT_DONE } 132 } 133 134 /// 获取当前进程的pcb 135 pub fn current_pcb() -> Arc<ProcessControlBlock> { 136 if unlikely(unsafe { !__PROCESS_MANAGEMENT_INIT_DONE }) { 137 kerror!("unsafe__PROCESS_MANAGEMENT_INIT_DONE == false"); 138 loop { 139 spin_loop(); 140 } 141 } 142 return ProcessControlBlock::arch_current_pcb(); 143 } 144 145 /// 获取当前进程的pid 146 /// 147 /// 如果进程管理器未初始化完成，那么返回0 148 pub fn current_pid() -> Pid { 149 if unlikely(unsafe { !__PROCESS_MANAGEMENT_INIT_DONE }) { 150 return Pid(0); 151 } 152 153 return ProcessManager::current_pcb().pid(); 154 } 155 156 /// 增加当前进程的锁持有计数 157 #[inline(always)] 158 pub fn preempt_disable() { 159 if likely(unsafe { __PROCESS_MANAGEMENT_INIT_DONE }) { 160 ProcessManager::current_pcb().preempt_disable(); 161 } 162 } 163 164 /// 减少当前进程的锁持有计数 165 #[inline(always)] 166 pub fn preempt_enable() { 167 if likely(unsafe { __PROCESS_MANAGEMENT_INIT_DONE }) { 168 ProcessManager::current_pcb().preempt_enable(); 169 } 170 } 171 172 /// 根据pid获取进程的pcb 173 /// 174 /// ## 参数 175 /// 176 /// - `pid` : 进程的pid 177 /// 178 /// ## 返回值 179 /// 180 /// 如果找到了对应的进程，那么返回该进程的pcb，否则返回None 181 pub fn find(pid: Pid) -> Option<Arc<ProcessControlBlock>> { 182 return ALL_PROCESS.lock_irqsave().as_ref()?.get(&pid).cloned(); 183 } 184 185 /// 向系统中添加一个进程的pcb 186 /// 187 /// ## 参数 188 /// 189 /// - `pcb` : 进程的pcb 190 /// 191 /// ## 返回值 192 /// 193 /// 无 194 pub fn add_pcb(pcb: Arc<ProcessControlBlock>) { 195 ALL_PROCESS 196 .lock_irqsave() 197 .as_mut() 198 .unwrap() 199 .insert(pcb.pid(), pcb.clone()); 200 } 201 202 /// 唤醒一个进程 203 pub fn wakeup(pcb: &Arc<ProcessControlBlock>) -> Result<(), SystemError> { 204 let _guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; 205 let state = pcb.sched_info().inner_lock_read_irqsave().state(); 206 if state.is_blocked() { 207 let mut writer = pcb.sched_info().inner_lock_write_irqsave(); 208 let state = writer.state(); 209 if state.is_blocked() { 210 writer.set_state(ProcessState::Runnable); 211 // avoid deadlock 212 drop(writer); 213 214 sched_enqueue(pcb.clone(), true); 215 return Ok(()); 216 } else if state.is_exited() { 217 return Err(SystemError::EINVAL); 218 } else { 219 return Ok(()); 220 } 221 } else if state.is_exited() { 222 return Err(SystemError::EINVAL); 223 } else { 224 return Ok(()); 225 } 226 } 227 228 /// 唤醒暂停的进程 229 pub fn wakeup_stop(pcb: &Arc<ProcessControlBlock>) -> Result<(), SystemError> { 230 let _guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; 231 let state = pcb.sched_info().inner_lock_read_irqsave().state(); 232 if let ProcessState::Stopped = state { 233 let mut writer = pcb.sched_info().inner_lock_write_irqsave(); 234 let state = writer.state(); 235 if let ProcessState::Stopped = state { 236 writer.set_state(ProcessState::Runnable); 237 // avoid deadlock 238 drop(writer); 239 240 sched_enqueue(pcb.clone(), true); 241 return Ok(()); 242 } else if state.is_runnable() { 243 return Ok(()); 244 } else { 245 return Err(SystemError::EINVAL); 246 } 247 } else if state.is_runnable() { 248 return Ok(()); 249 } else { 250 return Err(SystemError::EINVAL); 251 } 252 } 253 254 /// 标志当前进程永久睡眠，但是发起调度的工作，应该由调用者完成 255 /// 256 /// ## 注意 257 /// 258 /// - 进入当前函数之前，不能持有sched_info的锁 259 /// - 进入当前函数之前，必须关闭中断 260 /// - 进入当前函数之后必须保证逻辑的正确性，避免被重复加入调度队列 261 pub fn mark_sleep(interruptable: bool) -> Result<(), SystemError> { 262 assert_eq!( 263 CurrentIrqArch::is_irq_enabled(), 264 false, 265 "interrupt must be disabled before enter ProcessManager::mark_sleep()" 266 ); 267 268 let pcb = ProcessManager::current_pcb(); 269 let mut writer = pcb.sched_info().inner_lock_write_irqsave(); 270 if !matches!(writer.state(), ProcessState::Exited(_)) { 271 writer.set_state(ProcessState::Blocked(interruptable)); 272 pcb.flags().insert(ProcessFlags::NEED_SCHEDULE); 273 drop(writer); 274 275 return Ok(()); 276 } 277 return Err(SystemError::EINTR); 278 } 279 280 /// 标志当前进程为停止状态，但是发起调度的工作，应该由调用者完成 281 /// 282 /// ## 注意 283 /// 284 /// - 进入当前函数之前，不能持有sched_info的锁 285 /// - 进入当前函数之前，必须关闭中断 286 pub fn mark_stop() -> Result<(), SystemError> { 287 assert_eq!( 288 CurrentIrqArch::is_irq_enabled(), 289 false, 290 "interrupt must be disabled before enter ProcessManager::mark_stop()" 291 ); 292 293 let pcb = ProcessManager::current_pcb(); 294 let mut writer = pcb.sched_info().inner_lock_write_irqsave(); 295 if !matches!(writer.state(), ProcessState::Exited(_)) { 296 writer.set_state(ProcessState::Stopped); 297 pcb.flags().insert(ProcessFlags::NEED_SCHEDULE); 298 drop(writer); 299 300 return Ok(()); 301 } 302 return Err(SystemError::EINTR); 303 } 304 /// 当子进程退出后向父进程发送通知 305 fn exit_notify() { 306 let current = ProcessManager::current_pcb(); 307 // 让INIT进程收养所有子进程 308 if current.pid() != Pid(1) { 309 unsafe { 310 current 311 .adopt_childen() 312 .unwrap_or_else(|e| panic!("adopte_childen failed: error: {e:?}")) 313 }; 314 let r = current.parent_pcb.read_irqsave().upgrade(); 315 if r.is_none() { 316 return; 317 } 318 let parent_pcb = r.unwrap(); 319 let r = Syscall::kill(parent_pcb.pid(), Signal::SIGCHLD as i32); 320 if r.is_err() { 321 kwarn!( 322 "failed to send kill signal to {:?}'s parent pcb {:?}", 323 current.pid(), 324 parent_pcb.pid() 325 ); 326 } 327 // todo: 这里需要向父进程发送SIGCHLD信号 328 // todo: 这里还需要根据线程组的信息，决定信号的发送 329 } 330 } 331 332 /// 退出当前进程 333 /// 334 /// ## 参数 335 /// 336 /// - `exit_code` : 进程的退出码 337 pub fn exit(exit_code: usize) -> ! { 338 // 关中断 339 unsafe { CurrentIrqArch::interrupt_disable() }; 340 let pcb = ProcessManager::current_pcb(); 341 pcb.sched_info 342 .inner_lock_write_irqsave() 343 .set_state(ProcessState::Exited(exit_code)); 344 pcb.wait_queue.wakeup(Some(ProcessState::Blocked(true))); 345 346 // 进行进程退出后的工作 347 let thread = pcb.thread.write_irqsave(); 348 if let Some(addr) = thread.set_child_tid { 349 unsafe { clear_user(addr, core::mem::size_of::<i32>()).expect("clear tid failed") }; 350 } 351 352 if let Some(addr) = thread.clear_child_tid { 353 if Arc::strong_count(&pcb.basic().user_vm().expect("User VM Not found")) > 1 { 354 let _ = 355 Futex::futex_wake(addr, FutexFlag::FLAGS_MATCH_NONE, 1, FUTEX_BITSET_MATCH_ANY); 356 } 357 unsafe { clear_user(addr, core::mem::size_of::<i32>()).expect("clear tid failed") }; 358 } 359 360 // 如果是vfork出来的进程，则需要处理completion 361 if thread.vfork_done.is_some() { 362 thread.vfork_done.as_ref().unwrap().complete_all(); 363 } 364 drop(thread); 365 unsafe { pcb.basic_mut().set_user_vm(None) }; 366 drop(pcb); 367 ProcessManager::exit_notify(); 368 unsafe { CurrentIrqArch::interrupt_enable() }; 369 370 sched(); 371 loop {} 372 } 373 374 pub unsafe fn release(pid: Pid) { 375 let pcb = ProcessManager::find(pid); 376 if pcb.is_some() { 377 // let pcb = pcb.unwrap(); 378 // 判断该pcb是否在全局没有任何引用 379 // TODO: 当前，pcb的Arc指针存在泄露问题，引用计数不正确，打算在接下来实现debug专用的Arc，方便调试，然后解决这个bug。 380 // 因此目前暂时注释掉，使得能跑 381 // if Arc::strong_count(&pcb) <= 2 { 382 // drop(pcb); 383 // ALL_PROCESS.lock().as_mut().unwrap().remove(&pid); 384 // } else { 385 // // 如果不为1就panic 386 // let msg = format!("pcb '{:?}' is still referenced, strong count={}",pcb.pid(), Arc::strong_count(&pcb)); 387 // kerror!("{}", msg); 388 // panic!() 389 // } 390 391 ALL_PROCESS.lock_irqsave().as_mut().unwrap().remove(&pid); 392 } 393 } 394 395 /// 上下文切换完成后的钩子函数 396 unsafe fn switch_finish_hook() { 397 // kdebug!("switch_finish_hook"); 398 let prev_pcb = SWITCH_RESULT 399 .as_mut() 400 .unwrap() 401 .get_mut() 402 .prev_pcb 403 .take() 404 .expect("prev_pcb is None"); 405 let next_pcb = SWITCH_RESULT 406 .as_mut() 407 .unwrap() 408 .get_mut() 409 .next_pcb 410 .take() 411 .expect("next_pcb is None"); 412 413 // 由于进程切换前使用了SpinLockGuard::leak()，所以这里需要手动释放锁 414 prev_pcb.arch_info.force_unlock(); 415 next_pcb.arch_info.force_unlock(); 416 } 417 418 /// 如果目标进程正在目标CPU上运行，那么就让这个cpu陷入内核态 419 /// 420 /// ## 参数 421 /// 422 /// - `pcb` : 进程的pcb 423 #[allow(dead_code)] 424 pub fn kick(pcb: &Arc<ProcessControlBlock>) { 425 ProcessManager::current_pcb().preempt_disable(); 426 let cpu_id = pcb.sched_info().on_cpu(); 427 428 if let Some(cpu_id) = cpu_id { 429 let cpu_id = cpu_id; 430 431 if pcb.pid() == CPU_EXECUTING.get(cpu_id) { 432 kick_cpu(cpu_id).expect("ProcessManager::kick(): Failed to kick cpu"); 433 } 434 } 435 436 ProcessManager::current_pcb().preempt_enable(); 437 } 438 } 439 440 /// 上下文切换的钩子函数,当这个函数return的时候,将会发生上下文切换 441 #[cfg(target_arch = "x86_64")] 442 #[inline(never)] 443 pub unsafe extern "sysv64" fn switch_finish_hook() { 444 ProcessManager::switch_finish_hook(); 445 } 446 #[cfg(target_arch = "riscv64")] 447 pub unsafe extern "C" fn switch_finish_hook() { 448 ProcessManager::switch_finish_hook(); 449 } 450 451 int_like!(Pid, AtomicPid, usize, AtomicUsize); 452 453 impl Pid { 454 pub fn to_string(&self) -> String { 455 self.0.to_string() 456 } 457 } 458 459 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 460 pub enum ProcessState { 461 /// The process is running on a CPU or in a run queue. 462 Runnable, 463 /// The process is waiting for an event to occur. 464 /// 其中的bool表示该等待过程是否可以被打断。 465 /// - 如果该bool为true,那么，硬件中断/信号/其他系统事件都可以打断该等待过程，使得该进程重新进入Runnable状态。 466 /// - 如果该bool为false,那么，这个进程必须被显式的唤醒，才能重新进入Runnable状态。 467 Blocked(bool), 468 /// 进程被信号终止 469 Stopped, 470 /// 进程已经退出，usize表示进程的退出码 471 Exited(usize), 472 } 473 474 #[allow(dead_code)] 475 impl ProcessState { 476 #[inline(always)] 477 pub fn is_runnable(&self) -> bool { 478 return matches!(self, ProcessState::Runnable); 479 } 480 481 #[inline(always)] 482 pub fn is_blocked(&self) -> bool { 483 return matches!(self, ProcessState::Blocked(_)); 484 } 485 486 #[inline(always)] 487 pub fn is_blocked_interruptable(&self) -> bool { 488 return matches!(self, ProcessState::Blocked(true)); 489 } 490 491 /// Returns `true` if the process state is [`Exited`]. 492 #[inline(always)] 493 pub fn is_exited(&self) -> bool { 494 return matches!(self, ProcessState::Exited(_)); 495 } 496 497 /// Returns `true` if the process state is [`Stopped`]. 498 /// 499 /// [`Stopped`]: ProcessState::Stopped 500 #[inline(always)] 501 pub fn is_stopped(&self) -> bool { 502 matches!(self, ProcessState::Stopped) 503 } 504 505 /// Returns exit code if the process state is [`Exited`]. 506 #[inline(always)] 507 pub fn exit_code(&self) -> Option<usize> { 508 match self { 509 ProcessState::Exited(code) => Some(*code), 510 _ => None, 511 } 512 } 513 } 514 515 bitflags! { 516 /// pcb的标志位 517 pub struct ProcessFlags: usize { 518 /// 当前pcb表示一个内核线程 519 const KTHREAD = 1 << 0; 520 /// 当前进程需要被调度 521 const NEED_SCHEDULE = 1 << 1; 522 /// 进程由于vfork而与父进程存在资源共享 523 const VFORK = 1 << 2; 524 /// 进程不可被冻结 525 const NOFREEZE = 1 << 3; 526 /// 进程正在退出 527 const EXITING = 1 << 4; 528 /// 进程由于接收到终止信号唤醒 529 const WAKEKILL = 1 << 5; 530 /// 进程由于接收到信号而退出.(Killed by a signal) 531 const SIGNALED = 1 << 6; 532 /// 进程需要迁移到其他cpu上 533 const NEED_MIGRATE = 1 << 7; 534 /// 随机化的虚拟地址空间，主要用于动态链接器的加载 535 const RANDOMIZE = 1 << 8; 536 } 537 } 538 539 #[derive(Debug)] 540 pub struct ProcessControlBlock { 541 /// 当前进程的pid 542 pid: Pid, 543 /// 当前进程的线程组id（这个值在同一个线程组内永远不变） 544 tgid: Pid, 545 546 basic: RwLock<ProcessBasicInfo>, 547 /// 当前进程的自旋锁持有计数 548 preempt_count: AtomicUsize, 549 550 flags: LockFreeFlags<ProcessFlags>, 551 worker_private: SpinLock<Option<WorkerPrivate>>, 552 /// 进程的内核栈 553 kernel_stack: RwLock<KernelStack>, 554 555 /// 系统调用栈 556 syscall_stack: RwLock<KernelStack>, 557 558 /// 与调度相关的信息 559 sched_info: ProcessSchedulerInfo, 560 /// 与处理器架构相关的信息 561 arch_info: SpinLock<ArchPCBInfo>, 562 /// 与信号处理相关的信息(似乎可以是无锁的) 563 sig_info: RwLock<ProcessSignalInfo>, 564 /// 信号处理结构体 565 sig_struct: SpinLock<SignalStruct>, 566 /// 退出信号S 567 exit_signal: AtomicSignal, 568 569 /// 父进程指针 570 parent_pcb: RwLock<Weak<ProcessControlBlock>>, 571 /// 真实父进程指针 572 real_parent_pcb: RwLock<Weak<ProcessControlBlock>>, 573 574 /// 子进程链表 575 children: RwLock<Vec<Pid>>, 576 577 /// 等待队列 578 wait_queue: WaitQueue, 579 580 /// 线程信息 581 thread: RwLock<ThreadInfo>, 582 } 583 584 impl ProcessControlBlock { 585 /// Generate a new pcb. 586 /// 587 /// ## 参数 588 /// 589 /// - `name` : 进程的名字 590 /// - `kstack` : 进程的内核栈 591 /// 592 /// ## 返回值 593 /// 594 /// 返回一个新的pcb 595 pub fn new(name: String, kstack: KernelStack) -> Arc<Self> { 596 return Self::do_create_pcb(name, kstack, false); 597 } 598 599 /// 创建一个新的idle进程 600 /// 601 /// 请注意，这个函数只能在进程管理初始化的时候调用。 602 pub fn new_idle(cpu_id: u32, kstack: KernelStack) -> Arc<Self> { 603 let name = format!("idle-{}", cpu_id); 604 return Self::do_create_pcb(name, kstack, true); 605 } 606 607 #[inline(never)] 608 fn do_create_pcb(name: String, kstack: KernelStack, is_idle: bool) -> Arc<Self> { 609 let (pid, ppid, cwd) = if is_idle { 610 (Pid(0), Pid(0), "/".to_string()) 611 } else { 612 let ppid = ProcessManager::current_pcb().pid(); 613 let cwd = ProcessManager::current_pcb().basic().cwd(); 614 (Self::generate_pid(), ppid, cwd) 615 }; 616 617 let basic_info = ProcessBasicInfo::new(Pid(0), ppid, name, cwd, None); 618 let preempt_count = AtomicUsize::new(0); 619 let flags = unsafe { LockFreeFlags::new(ProcessFlags::empty()) }; 620 621 let sched_info = ProcessSchedulerInfo::new(None); 622 let arch_info = SpinLock::new(ArchPCBInfo::new(&kstack)); 623 624 let ppcb: Weak<ProcessControlBlock> = ProcessManager::find(ppid) 625 .map(|p| Arc::downgrade(&p)) 626 .unwrap_or_else(|| Weak::new()); 627 628 let pcb = Self { 629 pid, 630 tgid: pid, 631 basic: basic_info, 632 preempt_count, 633 flags, 634 kernel_stack: RwLock::new(kstack), 635 syscall_stack: RwLock::new(KernelStack::new().unwrap()), 636 worker_private: SpinLock::new(None), 637 sched_info, 638 arch_info, 639 sig_info: RwLock::new(ProcessSignalInfo::default()), 640 sig_struct: SpinLock::new(SignalStruct::new()), 641 exit_signal: AtomicSignal::new(Signal::SIGCHLD), 642 parent_pcb: RwLock::new(ppcb.clone()), 643 real_parent_pcb: RwLock::new(ppcb), 644 children: RwLock::new(Vec::new()), 645 wait_queue: WaitQueue::INIT, 646 thread: RwLock::new(ThreadInfo::new()), 647 }; 648 649 // 初始化系统调用栈 650 #[cfg(target_arch = "x86_64")] 651 pcb.arch_info 652 .lock() 653 .init_syscall_stack(&pcb.syscall_stack.read()); 654 655 let pcb = Arc::new(pcb); 656 657 // 设置进程的arc指针到内核栈和系统调用栈的最低地址处 658 unsafe { 659 pcb.kernel_stack 660 .write() 661 .set_pcb(Arc::downgrade(&pcb)) 662 .unwrap(); 663 664 pcb.syscall_stack 665 .write() 666 .set_pcb(Arc::downgrade(&pcb)) 667 .unwrap() 668 }; 669 670 // 将当前pcb加入父进程的子进程哈希表中 671 if pcb.pid() > Pid(1) { 672 if let Some(ppcb_arc) = pcb.parent_pcb.read_irqsave().upgrade() { 673 let mut children = ppcb_arc.children.write_irqsave(); 674 children.push(pcb.pid()); 675 } else { 676 panic!("parent pcb is None"); 677 } 678 } 679 680 return pcb; 681 } 682 683 /// 生成一个新的pid 684 #[inline(always)] 685 fn generate_pid() -> Pid { 686 static NEXT_PID: AtomicPid = AtomicPid::new(Pid(1)); 687 return NEXT_PID.fetch_add(Pid(1), Ordering::SeqCst); 688 } 689 690 /// 返回当前进程的锁持有计数 691 #[inline(always)] 692 pub fn preempt_count(&self) -> usize { 693 return self.preempt_count.load(Ordering::SeqCst); 694 } 695 696 /// 增加当前进程的锁持有计数 697 #[inline(always)] 698 pub fn preempt_disable(&self) { 699 self.preempt_count.fetch_add(1, Ordering::SeqCst); 700 } 701 702 /// 减少当前进程的锁持有计数 703 #[inline(always)] 704 pub fn preempt_enable(&self) { 705 self.preempt_count.fetch_sub(1, Ordering::SeqCst); 706 } 707 708 #[inline(always)] 709 pub unsafe fn set_preempt_count(&self, count: usize) { 710 self.preempt_count.store(count, Ordering::SeqCst); 711 } 712 713 #[inline(always)] 714 pub fn flags(&self) -> &mut ProcessFlags { 715 return self.flags.get_mut(); 716 } 717 718 /// 请注意，这个值能在中断上下文中读取，但不能被中断上下文修改 719 /// 否则会导致死锁 720 #[inline(always)] 721 pub fn basic(&self) -> RwLockReadGuard<ProcessBasicInfo> { 722 return self.basic.read_irqsave(); 723 } 724 725 #[inline(always)] 726 pub fn set_name(&self, name: String) { 727 self.basic.write().set_name(name); 728 } 729 730 #[inline(always)] 731 pub fn basic_mut(&self) -> RwLockWriteGuard<ProcessBasicInfo> { 732 return self.basic.write_irqsave(); 733 } 734 735 /// # 获取arch info的锁，同时关闭中断 736 #[inline(always)] 737 pub fn arch_info_irqsave(&self) -> SpinLockGuard<ArchPCBInfo> { 738 return self.arch_info.lock_irqsave(); 739 } 740 741 /// # 获取arch info的锁，但是不关闭中断 742 /// 743 /// 由于arch info在进程切换的时候会使用到， 744 /// 因此在中断上下文外，获取arch info 而不irqsave是不安全的. 745 /// 746 /// 只能在以下情况下使用这个函数： 747 /// - 在中断上下文中（中断已经禁用），获取arch info的锁。 748 /// - 刚刚创建新的pcb 749 #[inline(always)] 750 pub unsafe fn arch_info(&self) -> SpinLockGuard<ArchPCBInfo> { 751 return self.arch_info.lock(); 752 } 753 754 #[inline(always)] 755 pub fn kernel_stack(&self) -> RwLockReadGuard<KernelStack> { 756 return self.kernel_stack.read(); 757 } 758 759 #[inline(always)] 760 #[allow(dead_code)] 761 pub fn kernel_stack_mut(&self) -> RwLockWriteGuard<KernelStack> { 762 return self.kernel_stack.write(); 763 } 764 765 #[inline(always)] 766 pub fn sched_info(&self) -> &ProcessSchedulerInfo { 767 return &self.sched_info; 768 } 769 770 #[inline(always)] 771 pub fn worker_private(&self) -> SpinLockGuard<Option<WorkerPrivate>> { 772 return self.worker_private.lock(); 773 } 774 775 #[inline(always)] 776 pub fn pid(&self) -> Pid { 777 return self.pid; 778 } 779 780 #[inline(always)] 781 pub fn tgid(&self) -> Pid { 782 return self.tgid; 783 } 784 785 /// 获取文件描述符表的Arc指针 786 #[inline(always)] 787 pub fn fd_table(&self) -> Arc<RwLock<FileDescriptorVec>> { 788 return self.basic.read().fd_table().unwrap(); 789 } 790 791 /// 根据文件描述符序号，获取socket对象的Arc指针 792 /// 793 /// ## 参数 794 /// 795 /// - `fd` 文件描述符序号 796 /// 797 /// ## 返回值 798 /// 799 /// Option(&mut Box<dyn Socket>) socket对象的可变引用. 如果文件描述符不是socket，那么返回None 800 pub fn get_socket(&self, fd: i32) -> Option<Arc<SocketInode>> { 801 let binding = ProcessManager::current_pcb().fd_table(); 802 let fd_table_guard = binding.read(); 803 804 let f = fd_table_guard.get_file_by_fd(fd)?; 805 drop(fd_table_guard); 806 807 let guard = f.lock(); 808 if guard.file_type() != FileType::Socket { 809 return None; 810 } 811 let socket: Arc<SocketInode> = guard 812 .inode() 813 .downcast_arc::<SocketInode>() 814 .expect("Not a socket inode"); 815 return Some(socket); 816 } 817 818 /// 当前进程退出时,让初始进程收养所有子进程 819 unsafe fn adopt_childen(&self) -> Result<(), SystemError> { 820 match ProcessManager::find(Pid(1)) { 821 Some(init_pcb) => { 822 let childen_guard = self.children.write(); 823 let mut init_childen_guard = init_pcb.children.write(); 824 825 childen_guard.iter().for_each(|pid| { 826 init_childen_guard.push(*pid); 827 }); 828 829 return Ok(()); 830 } 831 _ => Err(SystemError::ECHILD), 832 } 833 } 834 835 /// 生成进程的名字 836 pub fn generate_name(program_path: &str, args: &Vec<String>) -> String { 837 let mut name = program_path.to_string(); 838 for arg in args { 839 name.push(' '); 840 name.push_str(arg); 841 } 842 return name; 843 } 844 845 pub fn sig_info_irqsave(&self) -> RwLockReadGuard<ProcessSignalInfo> { 846 self.sig_info.read_irqsave() 847 } 848 849 pub fn try_siginfo_irqsave(&self, times: u8) -> Option<RwLockReadGuard<ProcessSignalInfo>> { 850 for _ in 0..times { 851 if let Some(r) = self.sig_info.try_read_irqsave() { 852 return Some(r); 853 } 854 } 855 856 return None; 857 } 858 859 pub fn sig_info_mut(&self) -> RwLockWriteGuard<ProcessSignalInfo> { 860 self.sig_info.write_irqsave() 861 } 862 863 pub fn try_siginfo_mut(&self, times: u8) -> Option<RwLockWriteGuard<ProcessSignalInfo>> { 864 for _ in 0..times { 865 if let Some(r) = self.sig_info.try_write_irqsave() { 866 return Some(r); 867 } 868 } 869 870 return None; 871 } 872 873 pub fn sig_struct(&self) -> SpinLockGuard<SignalStruct> { 874 self.sig_struct.lock_irqsave() 875 } 876 877 pub fn try_sig_struct_irqsave(&self, times: u8) -> Option<SpinLockGuard<SignalStruct>> { 878 for _ in 0..times { 879 if let Ok(r) = self.sig_struct.try_lock_irqsave() { 880 return Some(r); 881 } 882 } 883 884 return None; 885 } 886 887 pub fn sig_struct_irqsave(&self) -> SpinLockGuard<SignalStruct> { 888 self.sig_struct.lock_irqsave() 889 } 890 } 891 892 impl Drop for ProcessControlBlock { 893 fn drop(&mut self) { 894 let irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; 895 // 在ProcFS中,解除进程的注册 896 procfs_unregister_pid(self.pid()) 897 .unwrap_or_else(|e| panic!("procfs_unregister_pid failed: error: {e:?}")); 898 899 if let Some(ppcb) = self.parent_pcb.read_irqsave().upgrade() { 900 ppcb.children 901 .write_irqsave() 902 .retain(|pid| *pid != self.pid()); 903 } 904 905 drop(irq_guard); 906 } 907 } 908 909 /// 线程信息 910 #[derive(Debug)] 911 pub struct ThreadInfo { 912 // 来自用户空间记录用户线程id的地址，在该线程结束时将该地址置0以通知父进程 913 clear_child_tid: Option<VirtAddr>, 914 set_child_tid: Option<VirtAddr>, 915 916 vfork_done: Option<Arc<Completion>>, 917 /// 线程组的组长 918 group_leader: Weak<ProcessControlBlock>, 919 } 920 921 impl ThreadInfo { 922 pub fn new() -> Self { 923 Self { 924 clear_child_tid: None, 925 set_child_tid: None, 926 vfork_done: None, 927 group_leader: Weak::default(), 928 } 929 } 930 931 pub fn group_leader(&self) -> Option<Arc<ProcessControlBlock>> { 932 return self.group_leader.upgrade(); 933 } 934 } 935 936 /// 进程的基本信息 937 /// 938 /// 这个结构体保存进程的基本信息，主要是那些不会随着进程的运行而经常改变的信息。 939 #[derive(Debug)] 940 pub struct ProcessBasicInfo { 941 /// 当前进程的进程组id 942 pgid: Pid, 943 /// 当前进程的父进程的pid 944 ppid: Pid, 945 /// 进程的名字 946 name: String, 947 948 /// 当前进程的工作目录 949 cwd: String, 950 951 /// 用户地址空间 952 user_vm: Option<Arc<AddressSpace>>, 953 954 /// 文件描述符表 955 fd_table: Option<Arc<RwLock<FileDescriptorVec>>>, 956 } 957 958 impl ProcessBasicInfo { 959 #[inline(never)] 960 pub fn new( 961 pgid: Pid, 962 ppid: Pid, 963 name: String, 964 cwd: String, 965 user_vm: Option<Arc<AddressSpace>>, 966 ) -> RwLock<Self> { 967 let fd_table = Arc::new(RwLock::new(FileDescriptorVec::new())); 968 return RwLock::new(Self { 969 pgid, 970 ppid, 971 name, 972 cwd, 973 user_vm, 974 fd_table: Some(fd_table), 975 }); 976 } 977 978 pub fn pgid(&self) -> Pid { 979 return self.pgid; 980 } 981 982 pub fn ppid(&self) -> Pid { 983 return self.ppid; 984 } 985 986 pub fn name(&self) -> &str { 987 return &self.name; 988 } 989 990 pub fn set_name(&mut self, name: String) { 991 self.name = name; 992 } 993 994 pub fn cwd(&self) -> String { 995 return self.cwd.clone(); 996 } 997 pub fn set_cwd(&mut self, path: String) { 998 return self.cwd = path; 999 } 1000 1001 pub fn user_vm(&self) -> Option<Arc<AddressSpace>> { 1002 return self.user_vm.clone(); 1003 } 1004 1005 pub unsafe fn set_user_vm(&mut self, user_vm: Option<Arc<AddressSpace>>) { 1006 self.user_vm = user_vm; 1007 } 1008 1009 pub fn fd_table(&self) -> Option<Arc<RwLock<FileDescriptorVec>>> { 1010 return self.fd_table.clone(); 1011 } 1012 1013 pub fn set_fd_table(&mut self, fd_table: Option<Arc<RwLock<FileDescriptorVec>>>) { 1014 self.fd_table = fd_table; 1015 } 1016 } 1017 1018 #[derive(Debug)] 1019 pub struct ProcessSchedulerInfo { 1020 /// 当前进程所在的cpu 1021 on_cpu: AtomicProcessorId, 1022 /// 如果当前进程等待被迁移到另一个cpu核心上（也就是flags中的PF_NEED_MIGRATE被置位）， 1023 /// 该字段存储要被迁移到的目标处理器核心号 1024 migrate_to: AtomicProcessorId, 1025 inner_locked: RwLock<InnerSchedInfo>, 1026 /// 进程的调度优先级 1027 priority: SchedPriority, 1028 /// 当前进程的虚拟运行时间 1029 virtual_runtime: AtomicIsize, 1030 /// 由实时调度器管理的时间片 1031 rt_time_slice: AtomicIsize, 1032 } 1033 1034 #[derive(Debug)] 1035 pub struct InnerSchedInfo { 1036 /// 当前进程的状态 1037 state: ProcessState, 1038 /// 进程的调度策略 1039 sched_policy: SchedPolicy, 1040 } 1041 1042 impl InnerSchedInfo { 1043 pub fn state(&self) -> ProcessState { 1044 return self.state; 1045 } 1046 1047 pub fn set_state(&mut self, state: ProcessState) { 1048 self.state = state; 1049 } 1050 1051 pub fn policy(&self) -> SchedPolicy { 1052 return self.sched_policy; 1053 } 1054 } 1055 1056 impl ProcessSchedulerInfo { 1057 #[inline(never)] 1058 pub fn new(on_cpu: Option<ProcessorId>) -> Self { 1059 let cpu_id = on_cpu.unwrap_or(ProcessorId::INVALID); 1060 return Self { 1061 on_cpu: AtomicProcessorId::new(cpu_id), 1062 migrate_to: AtomicProcessorId::new(ProcessorId::INVALID), 1063 inner_locked: RwLock::new(InnerSchedInfo { 1064 state: ProcessState::Blocked(false), 1065 sched_policy: SchedPolicy::CFS, 1066 }), 1067 virtual_runtime: AtomicIsize::new(0), 1068 rt_time_slice: AtomicIsize::new(0), 1069 priority: SchedPriority::new(100).unwrap(), 1070 }; 1071 } 1072 1073 pub fn on_cpu(&self) -> Option<ProcessorId> { 1074 let on_cpu = self.on_cpu.load(Ordering::SeqCst); 1075 if on_cpu == ProcessorId::INVALID { 1076 return None; 1077 } else { 1078 return Some(on_cpu); 1079 } 1080 } 1081 1082 pub fn set_on_cpu(&self, on_cpu: Option<ProcessorId>) { 1083 if let Some(cpu_id) = on_cpu { 1084 self.on_cpu.store(cpu_id, Ordering::SeqCst); 1085 } else { 1086 self.on_cpu.store(ProcessorId::INVALID, Ordering::SeqCst); 1087 } 1088 } 1089 1090 pub fn migrate_to(&self) -> Option<ProcessorId> { 1091 let migrate_to = self.migrate_to.load(Ordering::SeqCst); 1092 if migrate_to == ProcessorId::INVALID { 1093 return None; 1094 } else { 1095 return Some(migrate_to); 1096 } 1097 } 1098 1099 pub fn set_migrate_to(&self, migrate_to: Option<ProcessorId>) { 1100 if let Some(data) = migrate_to { 1101 self.migrate_to.store(data, Ordering::SeqCst); 1102 } else { 1103 self.migrate_to 1104 .store(ProcessorId::INVALID, Ordering::SeqCst) 1105 } 1106 } 1107 1108 pub fn inner_lock_write_irqsave(&self) -> RwLockWriteGuard<InnerSchedInfo> { 1109 return self.inner_locked.write_irqsave(); 1110 } 1111 1112 pub fn inner_lock_read_irqsave(&self) -> RwLockReadGuard<InnerSchedInfo> { 1113 return self.inner_locked.read_irqsave(); 1114 } 1115 1116 pub fn inner_lock_try_read_irqsave( 1117 &self, 1118 times: u8, 1119 ) -> Option<RwLockReadGuard<InnerSchedInfo>> { 1120 for _ in 0..times { 1121 if let Some(r) = self.inner_locked.try_read_irqsave() { 1122 return Some(r); 1123 } 1124 } 1125 1126 return None; 1127 } 1128 1129 pub fn inner_lock_try_upgradable_read_irqsave( 1130 &self, 1131 times: u8, 1132 ) -> Option<RwLockUpgradableGuard<InnerSchedInfo>> { 1133 for _ in 0..times { 1134 if let Some(r) = self.inner_locked.try_upgradeable_read_irqsave() { 1135 return Some(r); 1136 } 1137 } 1138 1139 return None; 1140 } 1141 1142 pub fn virtual_runtime(&self) -> isize { 1143 return self.virtual_runtime.load(Ordering::SeqCst); 1144 } 1145 1146 pub fn set_virtual_runtime(&self, virtual_runtime: isize) { 1147 self.virtual_runtime 1148 .store(virtual_runtime, Ordering::SeqCst); 1149 } 1150 pub fn increase_virtual_runtime(&self, delta: isize) { 1151 self.virtual_runtime.fetch_add(delta, Ordering::SeqCst); 1152 } 1153 1154 pub fn rt_time_slice(&self) -> isize { 1155 return self.rt_time_slice.load(Ordering::SeqCst); 1156 } 1157 1158 pub fn set_rt_time_slice(&self, rt_time_slice: isize) { 1159 self.rt_time_slice.store(rt_time_slice, Ordering::SeqCst); 1160 } 1161 1162 pub fn increase_rt_time_slice(&self, delta: isize) { 1163 self.rt_time_slice.fetch_add(delta, Ordering::SeqCst); 1164 } 1165 1166 pub fn priority(&self) -> SchedPriority { 1167 return self.priority; 1168 } 1169 } 1170 1171 #[derive(Debug, Clone)] 1172 pub struct KernelStack { 1173 stack: Option<AlignedBox<[u8; KernelStack::SIZE], { KernelStack::ALIGN }>>, 1174 /// 标记该内核栈是否可以被释放 1175 can_be_freed: bool, 1176 } 1177 1178 impl KernelStack { 1179 pub const SIZE: usize = 0x4000; 1180 pub const ALIGN: usize = 0x4000; 1181 1182 pub fn new() -> Result<Self, SystemError> { 1183 return Ok(Self { 1184 stack: Some( 1185 AlignedBox::<[u8; KernelStack::SIZE], { KernelStack::ALIGN }>::new_zeroed()?, 1186 ), 1187 can_be_freed: true, 1188 }); 1189 } 1190 1191 /// 根据已有的空间，构造一个内核栈结构体 1192 /// 1193 /// 仅仅用于BSP启动时，为idle进程构造内核栈。其他时候使用这个函数，很可能造成错误！ 1194 pub unsafe fn from_existed(base: VirtAddr) -> Result<Self, SystemError> { 1195 if base.is_null() || base.check_aligned(Self::ALIGN) == false { 1196 return Err(SystemError::EFAULT); 1197 } 1198 1199 return Ok(Self { 1200 stack: Some( 1201 AlignedBox::<[u8; KernelStack::SIZE], { KernelStack::ALIGN }>::new_unchecked( 1202 base.data() as *mut [u8; KernelStack::SIZE], 1203 ), 1204 ), 1205 can_be_freed: false, 1206 }); 1207 } 1208 1209 /// 返回内核栈的起始虚拟地址(低地址) 1210 pub fn start_address(&self) -> VirtAddr { 1211 return VirtAddr::new(self.stack.as_ref().unwrap().as_ptr() as usize); 1212 } 1213 1214 /// 返回内核栈的结束虚拟地址(高地址)(不包含该地址) 1215 pub fn stack_max_address(&self) -> VirtAddr { 1216 return VirtAddr::new(self.stack.as_ref().unwrap().as_ptr() as usize + Self::SIZE); 1217 } 1218 1219 pub unsafe fn set_pcb(&mut self, pcb: Weak<ProcessControlBlock>) -> Result<(), SystemError> { 1220 // 将一个Weak<ProcessControlBlock>放到内核栈的最低地址处 1221 let p: *const ProcessControlBlock = Weak::into_raw(pcb); 1222 let stack_bottom_ptr = self.start_address().data() as *mut *const ProcessControlBlock; 1223 1224 // 如果内核栈的最低地址处已经有了一个pcb，那么，这里就不再设置,直接返回错误 1225 if unlikely(unsafe { !(*stack_bottom_ptr).is_null() }) { 1226 kerror!("kernel stack bottom is not null: {:p}", *stack_bottom_ptr); 1227 return Err(SystemError::EPERM); 1228 } 1229 // 将pcb的地址放到内核栈的最低地址处 1230 unsafe { 1231 *stack_bottom_ptr = p; 1232 } 1233 1234 return Ok(()); 1235 } 1236 1237 /// 清除内核栈的pcb指针 1238 /// 1239 /// ## 参数 1240 /// 1241 /// - `force` : 如果为true,那么，即使该内核栈的pcb指针不为null，也会被强制清除而不处理Weak指针问题 1242 pub unsafe fn clear_pcb(&mut self, force: bool) { 1243 let stack_bottom_ptr = self.start_address().data() as *mut *const ProcessControlBlock; 1244 if unlikely(unsafe { (*stack_bottom_ptr).is_null() }) { 1245 return; 1246 } 1247 1248 if !force { 1249 let pcb_ptr: Weak<ProcessControlBlock> = Weak::from_raw(*stack_bottom_ptr); 1250 drop(pcb_ptr); 1251 } 1252 1253 *stack_bottom_ptr = core::ptr::null(); 1254 } 1255 1256 /// 返回指向当前内核栈pcb的Arc指针 1257 #[allow(dead_code)] 1258 pub unsafe fn pcb(&self) -> Option<Arc<ProcessControlBlock>> { 1259 // 从内核栈的最低地址处取出pcb的地址 1260 let p = self.stack.as_ref().unwrap().as_ptr() as *const *const ProcessControlBlock; 1261 if unlikely(unsafe { (*p).is_null() }) { 1262 return None; 1263 } 1264 1265 // 为了防止内核栈的pcb指针被释放，这里需要将其包装一下，使得Arc的drop不会被调用 1266 let weak_wrapper: ManuallyDrop<Weak<ProcessControlBlock>> = 1267 ManuallyDrop::new(Weak::from_raw(*p)); 1268 1269 let new_arc: Arc<ProcessControlBlock> = weak_wrapper.upgrade()?; 1270 return Some(new_arc); 1271 } 1272 } 1273 1274 impl Drop for KernelStack { 1275 fn drop(&mut self) { 1276 if !self.stack.is_none() { 1277 let ptr = self.stack.as_ref().unwrap().as_ptr() as *const *const ProcessControlBlock; 1278 if unsafe { !(*ptr).is_null() } { 1279 let pcb_ptr: Weak<ProcessControlBlock> = unsafe { Weak::from_raw(*ptr) }; 1280 drop(pcb_ptr); 1281 } 1282 } 1283 // 如果该内核栈不可以被释放，那么，这里就forget，不调用AlignedBox的drop函数 1284 if !self.can_be_freed { 1285 let bx = self.stack.take(); 1286 core::mem::forget(bx); 1287 } 1288 } 1289 } 1290 1291 pub fn process_init() { 1292 ProcessManager::init(); 1293 } 1294 1295 #[derive(Debug)] 1296 pub struct ProcessSignalInfo { 1297 // 当前进程 1298 sig_block: SigSet, 1299 // sig_pending 中存储当前线程要处理的信号 1300 sig_pending: SigPending, 1301 // sig_shared_pending 中存储当前线程所属进程要处理的信号 1302 sig_shared_pending: SigPending, 1303 // 当前进程对应的tty 1304 tty: Option<Arc<TtyCore>>, 1305 } 1306 1307 impl ProcessSignalInfo { 1308 pub fn sig_block(&self) -> &SigSet { 1309 &self.sig_block 1310 } 1311 1312 pub fn sig_pending(&self) -> &SigPending { 1313 &self.sig_pending 1314 } 1315 1316 pub fn sig_pending_mut(&mut self) -> &mut SigPending { 1317 &mut self.sig_pending 1318 } 1319 1320 pub fn sig_block_mut(&mut self) -> &mut SigSet { 1321 &mut self.sig_block 1322 } 1323 1324 pub fn sig_shared_pending_mut(&mut self) -> &mut SigPending { 1325 &mut self.sig_shared_pending 1326 } 1327 1328 pub fn sig_shared_pending(&self) -> &SigPending { 1329 &self.sig_shared_pending 1330 } 1331 1332 pub fn tty(&self) -> Option<Arc<TtyCore>> { 1333 self.tty.clone() 1334 } 1335 1336 pub fn set_tty(&mut self, tty: Arc<TtyCore>) { 1337 self.tty = Some(tty); 1338 } 1339 1340 /// 从 pcb 的 siginfo中取出下一个要处理的信号，先处理线程信号，再处理进程信号 1341 /// 1342 /// ## 参数 1343 /// 1344 /// - `sig_mask` 被忽略掉的信号 1345 /// 1346 pub fn dequeue_signal(&mut self, sig_mask: &SigSet) -> (Signal, Option<SigInfo>) { 1347 let res = self.sig_pending.dequeue_signal(sig_mask); 1348 if res.0 != Signal::INVALID { 1349 return res; 1350 } else { 1351 return self.sig_shared_pending.dequeue_signal(sig_mask); 1352 } 1353 } 1354 } 1355 1356 impl Default for ProcessSignalInfo { 1357 fn default() -> Self { 1358 Self { 1359 sig_block: SigSet::empty(), 1360 sig_pending: SigPending::default(), 1361 sig_shared_pending: SigPending::default(), 1362 tty: None, 1363 } 1364 } 1365 } 1366