169 static int expand_fdtable(struct files_struct *files, unsigned int nr)  in expand_fdtable()  argument
170 	__releases(files->file_lock)  in expand_fdtable()
171 	__acquires(files->file_lock)  in expand_fdtable()
175 	spin_unlock(&files->file_lock);  in expand_fdtable()
181 	if (atomic_read(&files->count) > 1)  in expand_fdtable()
184 	spin_lock(&files->file_lock);  in expand_fdtable()
195 	cur_fdt = files_fdtable(files);  in expand_fdtable()
198 	rcu_assign_pointer(files->fdt, new_fdt);  in expand_fdtable()
199 	if (cur_fdt != &files->fdtab)  in expand_fdtable()
214 static int expand_files(struct files_struct *files, unsigned int nr)  in expand_files()  argument
215 	__releases(files->file_lock)  in expand_files()
216 	__acquires(files->file_lock)  in expand_files()
222 	fdt = files_fdtable(files);  in expand_files()
232 	if (unlikely(files->resize_in_progress)) {  in expand_files()
233 		spin_unlock(&files->file_lock);  in expand_files()
235 		wait_event(files->resize_wait, !files->resize_in_progress);  in expand_files()
236 		spin_lock(&files->file_lock);  in expand_files()
241 	files->resize_in_progress = true;  in expand_files()
242 	expanded = expand_fdtable(files, nr);  in expand_files()
243 	files->resize_in_progress = false;  in expand_files()
245 	wake_up_all(&files->resize_wait);  in expand_files()
412 static struct fdtable *close_files(struct files_struct * files)  in close_files()  argument
419 	struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in close_files()
432 					filp_close(file, files);  in close_files()
444 void put_files_struct(struct files_struct *files)  in put_files_struct()  argument
446 	if (atomic_dec_and_test(&files->count)) {  in put_files_struct()
447 		struct fdtable *fdt = close_files(files);  in put_files_struct()
450 		if (fdt != &files->fdtab)  in put_files_struct()
452 		kmem_cache_free(files_cachep, files);  in put_files_struct()
458 	struct files_struct * files = tsk->files;  in exit_files()  local
460 	if (files) {  in exit_files()
462 		tsk->files = NULL;  in exit_files()
464 		put_files_struct(files);  in exit_files()
501 	struct files_struct *files = current->files;  in alloc_fd()  local
506 	spin_lock(&files->file_lock);  in alloc_fd()
508 	fdt = files_fdtable(files);  in alloc_fd()
510 	if (fd < files->next_fd)  in alloc_fd()
511 		fd = files->next_fd;  in alloc_fd()
524 	error = expand_files(files, fd);  in alloc_fd()
535 	if (start <= files->next_fd)  in alloc_fd()
536 		files->next_fd = fd + 1;  in alloc_fd()
553 	spin_unlock(&files->file_lock);  in alloc_fd()
568 static void __put_unused_fd(struct files_struct *files, unsigned int fd)  in __put_unused_fd()  argument
570 	struct fdtable *fdt = files_fdtable(files);  in __put_unused_fd()
572 	if (fd < files->next_fd)  in __put_unused_fd()
573 		files->next_fd = fd;  in __put_unused_fd()
578 	struct files_struct *files = current->files;  in put_unused_fd()  local
579 	spin_lock(&files->file_lock);  in put_unused_fd()
580 	__put_unused_fd(files, fd);  in put_unused_fd()
581 	spin_unlock(&files->file_lock);  in put_unused_fd()
604 	struct files_struct *files = current->files;  in fd_install()  local
609 	if (unlikely(files->resize_in_progress)) {  in fd_install()
611 		spin_lock(&files->file_lock);  in fd_install()
612 		fdt = files_fdtable(files);  in fd_install()
615 		spin_unlock(&files->file_lock);  in fd_install()
620 	fdt = rcu_dereference_sched(files->fdt);  in fd_install()
637 static struct file *pick_file(struct files_struct *files, unsigned fd)  in pick_file()  argument
639 	struct fdtable *fdt = files_fdtable(files);  in pick_file()
649 		__put_unused_fd(files, fd);  in pick_file()
656 	struct files_struct *files = current->files;  in close_fd()  local
659 	spin_lock(&files->file_lock);  in close_fd()
660 	file = pick_file(files, fd);  in close_fd()
661 	spin_unlock(&files->file_lock);  in close_fd()
665 	return filp_close(file, files);  in close_fd()
696 static inline void __range_close(struct files_struct *files, unsigned int fd,  in __range_close()  argument
702 	spin_lock(&files->file_lock);  in __range_close()
703 	n = last_fd(files_fdtable(files));  in __range_close()
707 		file = pick_file(files, fd);  in __range_close()
709 			spin_unlock(&files->file_lock);  in __range_close()
710 			filp_close(file, files);  in __range_close()
712 			spin_lock(&files->file_lock);  in __range_close()
714 			spin_unlock(&files->file_lock);  in __range_close()
716 			spin_lock(&files->file_lock);  in __range_close()
719 	spin_unlock(&files->file_lock);  in __range_close()
735 	struct files_struct *cur_fds = me->files, *fds = NULL;  in __close_range()
787 		me->files = cur_fds;  in __close_range()
801 	return pick_file(current->files, fd);  in __close_fd_get_file()
810 	struct files_struct *files = current->files;  in close_fd_get_file()  local
813 	spin_lock(&files->file_lock);  in close_fd_get_file()
814 	file = pick_file(files, fd);  in close_fd_get_file()
815 	spin_unlock(&files->file_lock);  in close_fd_get_file()
820 void do_close_on_exec(struct files_struct *files)  in do_close_on_exec()  argument
826 	spin_lock(&files->file_lock);  in do_close_on_exec()
830 		fdt = files_fdtable(files);  in do_close_on_exec()
845 			__put_unused_fd(files, fd);  in do_close_on_exec()
846 			spin_unlock(&files->file_lock);  in do_close_on_exec()
847 			filp_close(file, files);  in do_close_on_exec()
849 			spin_lock(&files->file_lock);  in do_close_on_exec()
853 	spin_unlock(&files->file_lock);  in do_close_on_exec()
856 static inline struct file *__fget_files_rcu(struct files_struct *files,  in __fget_files_rcu()  argument
861 		struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in __fget_files_rcu()
896 		if (unlikely(rcu_dereference_raw(files->fdt) != fdt) ||  in __fget_files_rcu()
910 static struct file *__fget_files(struct files_struct *files, unsigned int fd,  in __fget_files()  argument
916 	file = __fget_files_rcu(files, fd, mask);  in __fget_files()
924 	return __fget_files(current->files, fd, mask);  in __fget()
944 	if (task->files)  in fget_task()
945 		file = __fget_files(task->files, fd, 0);  in fget_task()
954 	struct files_struct *files;  in task_lookup_fd_rcu()  local
958 	files = task->files;  in task_lookup_fd_rcu()
959 	if (files)  in task_lookup_fd_rcu()
960 		file = files_lookup_fd_rcu(files, fd);  in task_lookup_fd_rcu()
969 	struct files_struct *files;  in task_lookup_next_fd_rcu()  local
974 	files = task->files;  in task_lookup_next_fd_rcu()
975 	if (files) {  in task_lookup_next_fd_rcu()
976 		for (; fd < files_fdtable(files)->max_fds; fd++) {  in task_lookup_next_fd_rcu()
977 			file = files_lookup_fd_rcu(files, fd);  in task_lookup_next_fd_rcu()
1006 	struct files_struct *files = current->files;  in __fget_light()  local
1018 	if (atomic_read_acquire(&files->count) == 1) {  in __fget_light()
1019 		file = files_lookup_fd_raw(files, fd);  in __fget_light()
1082 	struct files_struct *files = current->files;  in set_close_on_exec()  local
1084 	spin_lock(&files->file_lock);  in set_close_on_exec()
1085 	fdt = files_fdtable(files);  in set_close_on_exec()
1090 	spin_unlock(&files->file_lock);  in set_close_on_exec()
1095 	struct files_struct *files = current->files;  in get_close_on_exec()  local
1099 	fdt = files_fdtable(files);  in get_close_on_exec()
1105 static int do_dup2(struct files_struct *files,  in do_dup2()  argument
1107 __releases(&files->file_lock)  in do_dup2()
1126 	fdt = files_fdtable(files);  in do_dup2()
1137 	spin_unlock(&files->file_lock);  in do_dup2()
1140 		filp_close(tofree, files);  in do_dup2()
1145 	spin_unlock(&files->file_lock);  in do_dup2()
1152 	struct files_struct *files = current->files;  in replace_fd()  local
1160 	spin_lock(&files->file_lock);  in replace_fd()
1161 	err = expand_files(files, fd);  in replace_fd()
1164 	return do_dup2(files, file, fd, flags);  in replace_fd()
1167 	spin_unlock(&files->file_lock);  in replace_fd()
1236 	struct files_struct *files = current->files;  in ksys_dup3()  local
1247 	spin_lock(&files->file_lock);  in ksys_dup3()
1248 	err = expand_files(files, newfd);  in ksys_dup3()
1249 	file = files_lookup_fd_locked(files, oldfd);  in ksys_dup3()
1257 	return do_dup2(files, file, newfd, flags);  in ksys_dup3()
1262 	spin_unlock(&files->file_lock);  in ksys_dup3()
1274 		struct files_struct *files = current->files;  in SYSCALL_DEFINE2()  local
1278 		if (!files_lookup_fd_rcu(files, oldfd))  in SYSCALL_DEFINE2()
1315 int iterate_fd(struct files_struct *files, unsigned n,  in iterate_fd()  argument
1321 	if (!files)  in iterate_fd()
1323 	spin_lock(&files->file_lock);  in iterate_fd()
1324 	for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {  in iterate_fd()
1326 		file = rcu_dereference_check_fdtable(files, fdt->fd[n]);  in iterate_fd()
1333 	spin_unlock(&files->file_lock);  in iterate_fd()