34 static void __free_fdtable(struct fdtable *fdt)  in __free_fdtable()  argument
36 	kvfree(fdt->fd);  in __free_fdtable()
37 	kvfree(fdt->open_fds);  in __free_fdtable()
38 	kfree(fdt);  in __free_fdtable()
107 	struct fdtable *fdt;  in alloc_fdtable()  local
132 	fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);  in alloc_fdtable()
133 	if (!fdt)  in alloc_fdtable()
135 	fdt->max_fds = nr;  in alloc_fdtable()
139 	fdt->fd = data;  in alloc_fdtable()
146 	fdt->open_fds = data;  in alloc_fdtable()
148 	fdt->close_on_exec = data;  in alloc_fdtable()
150 	fdt->full_fds_bits = data;  in alloc_fdtable()
152 	return fdt;  in alloc_fdtable()
155 	kvfree(fdt->fd);  in alloc_fdtable()
157 	kfree(fdt);  in alloc_fdtable()
198 	rcu_assign_pointer(files->fdt, new_fdt);  in expand_fdtable()
218 	struct fdtable *fdt;  in expand_files()  local
222 	fdt = files_fdtable(files);  in expand_files()
225 	if (nr < fdt->max_fds)  in expand_files()
249 static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt)  in __set_close_on_exec()  argument
251 	__set_bit(fd, fdt->close_on_exec);  in __set_close_on_exec()
254 static inline void __clear_close_on_exec(unsigned int fd, struct fdtable *fdt)  in __clear_close_on_exec()  argument
256 	if (test_bit(fd, fdt->close_on_exec))  in __clear_close_on_exec()
257 		__clear_bit(fd, fdt->close_on_exec);  in __clear_close_on_exec()
260 static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)  in __set_open_fd()  argument
262 	__set_bit(fd, fdt->open_fds);  in __set_open_fd()
264 	if (!~fdt->open_fds[fd])  in __set_open_fd()
265 		__set_bit(fd, fdt->full_fds_bits);  in __set_open_fd()
268 static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)  in __clear_open_fd()  argument
270 	__clear_bit(fd, fdt->open_fds);  in __clear_open_fd()
271 	__clear_bit(fd / BITS_PER_LONG, fdt->full_fds_bits);  in __clear_open_fd()
274 static unsigned int count_open_files(struct fdtable *fdt)  in count_open_files()  argument
276 	unsigned int size = fdt->max_fds;  in count_open_files()
281 		if (fdt->open_fds[--i])  in count_open_files()
301 static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)  in sane_fdtable_size()  argument
305 	count = count_open_files(fdt);  in sane_fdtable_size()
402 	rcu_assign_pointer(newf->fdt, new_fdt);  in dup_fd()
419 	struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in close_files()  local
425 		if (i >= fdt->max_fds)  in close_files()
427 		set = fdt->open_fds[j++];  in close_files()
430 				struct file * file = xchg(&fdt->fd[i], NULL);  in close_files()
441 	return fdt;  in close_files()
447 		struct fdtable *fdt = close_files(files);  in put_files_struct()  local
450 		if (fdt != &files->fdtab)  in put_files_struct()
451 			__free_fdtable(fdt);  in put_files_struct()
470 	.fdt		= &init_files.fdtab,
482 static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)  in find_next_fd()  argument
484 	unsigned int maxfd = fdt->max_fds;  in find_next_fd()
488 	bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;  in find_next_fd()
493 	return find_next_zero_bit(fdt->open_fds, maxfd, start);  in find_next_fd()
504 	struct fdtable *fdt;  in alloc_fd()  local
508 	fdt = files_fdtable(files);  in alloc_fd()
513 	if (fd < fdt->max_fds)  in alloc_fd()
514 		fd = find_next_fd(fdt, fd);  in alloc_fd()
538 	__set_open_fd(fd, fdt);  in alloc_fd()
540 		__set_close_on_exec(fd, fdt);  in alloc_fd()
542 		__clear_close_on_exec(fd, fdt);  in alloc_fd()
546 	if (rcu_access_pointer(fdt->fd[fd]) != NULL) {  in alloc_fd()
548 		rcu_assign_pointer(fdt->fd[fd], NULL);  in alloc_fd()
570 	struct fdtable *fdt = files_fdtable(files);  in __put_unused_fd()  local
571 	__clear_open_fd(fd, fdt);  in __put_unused_fd()
605 	struct fdtable *fdt;  in fd_install()  local
612 		fdt = files_fdtable(files);  in fd_install()
613 		BUG_ON(fdt->fd[fd] != NULL);  in fd_install()
614 		rcu_assign_pointer(fdt->fd[fd], file);  in fd_install()
620 	fdt = rcu_dereference_sched(files->fdt);  in fd_install()
621 	BUG_ON(fdt->fd[fd] != NULL);  in fd_install()
622 	rcu_assign_pointer(fdt->fd[fd], file);  in fd_install()
639 	struct fdtable *fdt = files_fdtable(files);  in pick_file()  local
642 	if (fd >= fdt->max_fds)  in pick_file()
645 	file = fdt->fd[fd];  in pick_file()
647 		rcu_assign_pointer(fdt->fd[fd], NULL);  in pick_file()
676 static inline unsigned last_fd(struct fdtable *fdt)  in last_fd()  argument
678 	return fdt->max_fds - 1;  in last_fd()
684 	struct fdtable *fdt;  in __range_cloexec()  local
688 	fdt = files_fdtable(cur_fds);  in __range_cloexec()
689 	max_fd = min(last_fd(fdt), max_fd);  in __range_cloexec()
691 		bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);  in __range_cloexec()
820 	struct fdtable *fdt;  in do_close_on_exec()  local
827 		fdt = files_fdtable(files);  in do_close_on_exec()
828 		if (fd >= fdt->max_fds)  in do_close_on_exec()
830 		set = fdt->close_on_exec[i];  in do_close_on_exec()
833 		fdt->close_on_exec[i] = 0;  in do_close_on_exec()
838 			file = fdt->fd[fd];  in do_close_on_exec()
841 			rcu_assign_pointer(fdt->fd[fd], NULL);  in do_close_on_exec()
858 		struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in __fget_files_rcu()  local
861 		if (unlikely(fd >= fdt->max_fds))  in __fget_files_rcu()
864 		fdentry = fdt->fd + array_index_nospec(fd, fdt->max_fds);  in __fget_files_rcu()
893 		if (unlikely(rcu_dereference_raw(files->fdt) != fdt) ||  in __fget_files_rcu()
1066 	struct fdtable *fdt;  in set_close_on_exec()  local
1068 	fdt = files_fdtable(files);  in set_close_on_exec()
1070 		__set_close_on_exec(fd, fdt);  in set_close_on_exec()
1072 		__clear_close_on_exec(fd, fdt);  in set_close_on_exec()
1079 	struct fdtable *fdt;  in get_close_on_exec()  local
1082 	fdt = files_fdtable(files);  in get_close_on_exec()
1083 	res = close_on_exec(fd, fdt);  in get_close_on_exec()
1093 	struct fdtable *fdt;  in do_dup2()  local
1109 	fdt = files_fdtable(files);  in do_dup2()
1110 	tofree = fdt->fd[fd];  in do_dup2()
1111 	if (!tofree && fd_is_open(fd, fdt))  in do_dup2()
1114 	rcu_assign_pointer(fdt->fd[fd], file);  in do_dup2()
1115 	__set_open_fd(fd, fdt);  in do_dup2()
1117 		__set_close_on_exec(fd, fdt);  in do_dup2()
1119 		__clear_close_on_exec(fd, fdt);  in do_dup2()
1302 	struct fdtable *fdt;  in iterate_fd()  local
1307 	for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {  in iterate_fd()
1309 		file = rcu_dereference_check_fdtable(files, fdt->fd[n]);  in iterate_fd()