164 void __weak arch_release_task_struct(struct task_struct *tsk)  in arch_release_task_struct()  argument
176 static inline void free_task_struct(struct task_struct *tsk)  in free_task_struct()  argument
178 	kmem_cache_free(task_struct_cachep, tsk);  in free_task_struct()
225 static void thread_stack_delayed_free(struct task_struct *tsk)  in thread_stack_delayed_free()  argument
227 	struct vm_stack *vm_stack = tsk->stack;  in thread_stack_delayed_free()
229 	vm_stack->stack_vm_area = tsk->stack_vm_area;  in thread_stack_delayed_free()
272 static int alloc_thread_stack_node(struct task_struct *tsk, int node)  in alloc_thread_stack_node()  argument
299 		tsk->stack_vm_area = s;  in alloc_thread_stack_node()
300 		tsk->stack = stack;  in alloc_thread_stack_node()
327 	tsk->stack_vm_area = vm;  in alloc_thread_stack_node()
329 	tsk->stack = stack;  in alloc_thread_stack_node()
333 static void free_thread_stack(struct task_struct *tsk)  in free_thread_stack()  argument
335 	if (!try_release_thread_stack_to_cache(tsk->stack_vm_area))  in free_thread_stack()
336 		thread_stack_delayed_free(tsk);  in free_thread_stack()
338 	tsk->stack = NULL;  in free_thread_stack()
339 	tsk->stack_vm_area = NULL;  in free_thread_stack()
349 static void thread_stack_delayed_free(struct task_struct *tsk)  in thread_stack_delayed_free()  argument
351 	struct rcu_head *rh = tsk->stack;  in thread_stack_delayed_free()
356 static int alloc_thread_stack_node(struct task_struct *tsk, int node)  in alloc_thread_stack_node()  argument
362 		tsk->stack = kasan_reset_tag(page_address(page));  in alloc_thread_stack_node()
368 static void free_thread_stack(struct task_struct *tsk)  in free_thread_stack()  argument
370 	thread_stack_delayed_free(tsk);  in free_thread_stack()
371 	tsk->stack = NULL;  in free_thread_stack()
384 static void thread_stack_delayed_free(struct task_struct *tsk)  in thread_stack_delayed_free()  argument
386 	struct rcu_head *rh = tsk->stack;  in thread_stack_delayed_free()
391 static int alloc_thread_stack_node(struct task_struct *tsk, int node)  in alloc_thread_stack_node()  argument
396 	tsk->stack = stack;  in alloc_thread_stack_node()
400 static void free_thread_stack(struct task_struct *tsk)  in free_thread_stack()  argument
402 	thread_stack_delayed_free(tsk);  in free_thread_stack()
403 	tsk->stack = NULL;  in free_thread_stack()
417 static int alloc_thread_stack_node(struct task_struct *tsk, int node)  in alloc_thread_stack_node()  argument
421 	stack = arch_alloc_thread_stack_node(tsk, node);  in alloc_thread_stack_node()
422 	tsk->stack = stack;  in alloc_thread_stack_node()
426 static void free_thread_stack(struct task_struct *tsk)  in free_thread_stack()  argument
428 	arch_free_thread_stack(tsk);  in free_thread_stack()
429 	tsk->stack = NULL;  in free_thread_stack()
552 static void account_kernel_stack(struct task_struct *tsk, int account)  in account_kernel_stack()  argument
555 		struct vm_struct *vm = task_stack_vm_area(tsk);  in account_kernel_stack()
562 		void *stack = task_stack_page(tsk);  in account_kernel_stack()
570 void exit_task_stack_account(struct task_struct *tsk)  in exit_task_stack_account()  argument
572 	account_kernel_stack(tsk, -1);  in exit_task_stack_account()
578 		vm = task_stack_vm_area(tsk);  in exit_task_stack_account()
584 static void release_task_stack(struct task_struct *tsk)  in release_task_stack()  argument
586 	if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))  in release_task_stack()
589 	free_thread_stack(tsk);  in release_task_stack()
593 void put_task_stack(struct task_struct *tsk)  in put_task_stack()  argument
595 	if (refcount_dec_and_test(&tsk->stack_refcount))  in put_task_stack()
596 		release_task_stack(tsk);  in put_task_stack()
600 void free_task(struct task_struct *tsk)  in free_task()  argument
603 	WARN_ON_ONCE(tsk->seccomp.filter);  in free_task()
605 	release_user_cpus_ptr(tsk);  in free_task()
606 	scs_release(tsk);  in free_task()
613 	release_task_stack(tsk);  in free_task()
619 	WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);  in free_task()
621 	rt_mutex_debug_task_free(tsk);  in free_task()
622 	ftrace_graph_exit_task(tsk);  in free_task()
623 	arch_release_task_struct(tsk);  in free_task()
624 	if (tsk->flags & PF_KTHREAD)  in free_task()
625 		free_kthread_struct(tsk);  in free_task()
626 	bpf_task_storage_free(tsk);  in free_task()
627 	free_task_struct(tsk);  in free_task()
967 void __put_task_struct(struct task_struct *tsk)  in __put_task_struct()  argument
969 	WARN_ON(!tsk->exit_state);  in __put_task_struct()
970 	WARN_ON(refcount_read(&tsk->usage));  in __put_task_struct()
971 	WARN_ON(tsk == current);  in __put_task_struct()
973 	io_uring_free(tsk);  in __put_task_struct()
974 	cgroup_free(tsk);  in __put_task_struct()
975 	task_numa_free(tsk, true);  in __put_task_struct()
976 	security_task_free(tsk);  in __put_task_struct()
977 	exit_creds(tsk);  in __put_task_struct()
978 	delayacct_tsk_free(tsk);  in __put_task_struct()
979 	put_signal_struct(tsk->signal);  in __put_task_struct()
980 	sched_core_free(tsk);  in __put_task_struct()
981 	free_task(tsk);  in __put_task_struct()
1095 void set_task_stack_end_magic(struct task_struct *tsk)  in set_task_stack_end_magic()  argument
1099 	stackend = end_of_stack(tsk);  in set_task_stack_end_magic()
1105 	struct task_struct *tsk;  in dup_task_struct()  local
1110 	tsk = alloc_task_struct_node(node);  in dup_task_struct()
1111 	if (!tsk)  in dup_task_struct()
1114 	err = arch_dup_task_struct(tsk, orig);  in dup_task_struct()
1118 	err = alloc_thread_stack_node(tsk, node);  in dup_task_struct()
1123 	refcount_set(&tsk->stack_refcount, 1);  in dup_task_struct()
1125 	account_kernel_stack(tsk, 1);  in dup_task_struct()
1127 	err = scs_prepare(tsk, node);  in dup_task_struct()
1138 	tsk->seccomp.filter = NULL;  in dup_task_struct()
1141 	setup_thread_stack(tsk, orig);  in dup_task_struct()
1142 	clear_user_return_notifier(tsk);  in dup_task_struct()
1143 	clear_tsk_need_resched(tsk);  in dup_task_struct()
1144 	set_task_stack_end_magic(tsk);  in dup_task_struct()
1145 	clear_syscall_work_syscall_user_dispatch(tsk);  in dup_task_struct()
1148 	tsk->stack_canary = get_random_canary();  in dup_task_struct()
1151 		tsk->cpus_ptr = &tsk->cpus_mask;  in dup_task_struct()
1152 	dup_user_cpus_ptr(tsk, orig, node);  in dup_task_struct()
1158 	refcount_set(&tsk->rcu_users, 2);  in dup_task_struct()
1160 	refcount_set(&tsk->usage, 1);  in dup_task_struct()
1162 	tsk->btrace_seq = 0;  in dup_task_struct()
1164 	tsk->splice_pipe = NULL;  in dup_task_struct()
1165 	tsk->task_frag.page = NULL;  in dup_task_struct()
1166 	tsk->wake_q.next = NULL;  in dup_task_struct()
1167 	tsk->worker_private = NULL;  in dup_task_struct()
1169 	kcov_task_init(tsk);  in dup_task_struct()
1170 	kmsan_task_create(tsk);  in dup_task_struct()
1171 	kmap_local_fork(tsk);  in dup_task_struct()
1174 	tsk->fail_nth = 0;  in dup_task_struct()
1178 	tsk->throttle_disk = NULL;  in dup_task_struct()
1179 	tsk->use_memdelay = 0;  in dup_task_struct()
1183 	tsk->pasid_activated = 0;  in dup_task_struct()
1187 	tsk->active_memcg = NULL;  in dup_task_struct()
1191 	tsk->reported_split_lock = 0;  in dup_task_struct()
1195 	tsk->mm_cid = -1;  in dup_task_struct()
1196 	tsk->last_mm_cid = -1;  in dup_task_struct()
1197 	tsk->mm_cid_active = 0;  in dup_task_struct()
1198 	tsk->migrate_from_cpu = -1;  in dup_task_struct()
1200 	return tsk;  in dup_task_struct()
1203 	exit_task_stack_account(tsk);  in dup_task_struct()
1204 	free_thread_stack(tsk);  in dup_task_struct()
1206 	free_task_struct(tsk);  in dup_task_struct()
1570 static void complete_vfork_done(struct task_struct *tsk)  in complete_vfork_done()  argument
1574 	task_lock(tsk);  in complete_vfork_done()
1575 	vfork = tsk->vfork_done;  in complete_vfork_done()
1577 		tsk->vfork_done = NULL;  in complete_vfork_done()
1580 	task_unlock(tsk);  in complete_vfork_done()
1616 static void mm_release(struct task_struct *tsk, struct mm_struct *mm)  in mm_release()  argument
1618 	uprobe_free_utask(tsk);  in mm_release()
1621 	deactivate_mm(tsk, mm);  in mm_release()
1628 	if (tsk->clear_child_tid) {  in mm_release()
1634 			put_user(0, tsk->clear_child_tid);  in mm_release()
1635 			do_futex(tsk->clear_child_tid, FUTEX_WAKE,  in mm_release()
1638 		tsk->clear_child_tid = NULL;  in mm_release()
1645 	if (tsk->vfork_done)  in mm_release()
1646 		complete_vfork_done(tsk);  in mm_release()
1649 void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)  in exit_mm_release()  argument
1651 	futex_exit_release(tsk);  in exit_mm_release()
1652 	mm_release(tsk, mm);  in exit_mm_release()
1655 void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)  in exec_mm_release()  argument
1657 	futex_exec_release(tsk);  in exec_mm_release()
1658 	mm_release(tsk, mm);  in exec_mm_release()
1671 static struct mm_struct *dup_mm(struct task_struct *tsk,  in dup_mm()  argument
1683 	if (!mm_init(mm, tsk, mm->user_ns))  in dup_mm()
1708 static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)  in copy_mm()  argument
1712 	tsk->min_flt = tsk->maj_flt = 0;  in copy_mm()
1713 	tsk->nvcsw = tsk->nivcsw = 0;  in copy_mm()
1715 	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;  in copy_mm()
1716 	tsk->last_switch_time = 0;  in copy_mm()
1719 	tsk->mm = NULL;  in copy_mm()
1720 	tsk->active_mm = NULL;  in copy_mm()
1735 		mm = dup_mm(tsk, current->mm);  in copy_mm()
1740 	tsk->mm = mm;  in copy_mm()
1741 	tsk->active_mm = mm;  in copy_mm()
1742 	sched_mm_cid_fork(tsk);  in copy_mm()
1746 static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)  in copy_fs()  argument
1760 	tsk->fs = copy_fs_struct(fs);  in copy_fs()
1761 	if (!tsk->fs)  in copy_fs()
1766 static int copy_files(unsigned long clone_flags, struct task_struct *tsk,  in copy_files()  argument
1780 		tsk->files = NULL;  in copy_files()
1793 	tsk->files = newf;  in copy_files()
1799 static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)  in copy_sighand()  argument
1808 	RCU_INIT_POINTER(tsk->sighand, sig);  in copy_sighand()
1819 		flush_signal_handlers(tsk, 0);  in copy_sighand()
1848 static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)  in copy_signal()  argument
1856 	tsk->signal = sig;  in copy_signal()
1866 	sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);  in copy_signal()
1867 	tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);  in copy_signal()
1870 	sig->curr_target = tsk;  in copy_signal()
2188 	struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);  in __delayed_free_task()  local
2190 	free_task(tsk);  in __delayed_free_task()
2193 static __always_inline void delayed_free_task(struct task_struct *tsk)  in delayed_free_task()  argument
2196 		call_rcu(&tsk->rcu, __delayed_free_task);  in delayed_free_task()
2198 		free_task(tsk);  in delayed_free_task()
2201 static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)  in copy_oom_score_adj()  argument
2204 	if (!tsk->mm)  in copy_oom_score_adj()
2213 	set_bit(MMF_MULTIPROCESS, &tsk->mm->flags);  in copy_oom_score_adj()
2215 	tsk->signal->oom_score_adj = current->signal->oom_score_adj;  in copy_oom_score_adj()
2216 	tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;  in copy_oom_score_adj()