3  *  linux/fs/pipe.c
36  * New pipe buffers will be restricted to this size while the user is exceeding
37  * their pipe buffer quota. The general pipe use case needs at least two
39  * than two, then a write to a non-empty pipe may block even if the pipe is not
42  * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
44  * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
45  * own risk, namely: pipe writes to non-full pipes may block until the pipe is
51  * The max size that a non-root user is allowed to grow the pipe. Can
52  * be set by root in /proc/sys/fs/pipe-max-size
79 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)  in pipe_lock_nested()  argument
81 	if (pipe->files)  in pipe_lock_nested()
82 		mutex_lock_nested(&pipe->mutex, subclass);  in pipe_lock_nested()
85 void pipe_lock(struct pipe_inode_info *pipe)  in pipe_lock()  argument
88 	 * pipe_lock() nests non-pipe inode locks (for writing to a file)  in pipe_lock()
90 	pipe_lock_nested(pipe, I_MUTEX_PARENT);  in pipe_lock()
94 void pipe_unlock(struct pipe_inode_info *pipe)  in pipe_unlock()  argument
96 	if (pipe->files)  in pipe_unlock()
97 		mutex_unlock(&pipe->mutex);  in pipe_unlock()
101 static inline void __pipe_lock(struct pipe_inode_info *pipe)  in __pipe_lock()  argument
103 	mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);  in __pipe_lock()
106 static inline void __pipe_unlock(struct pipe_inode_info *pipe)  in __pipe_unlock()  argument
108 	mutex_unlock(&pipe->mutex);  in __pipe_unlock()
125 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,  in anon_pipe_buf_release()  argument
135 	if (page_count(page) == 1 && !pipe->tmp_page)  in anon_pipe_buf_release()
136 		pipe->tmp_page = page;  in anon_pipe_buf_release()
141 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,  in anon_pipe_buf_try_steal()  argument
155  * @pipe:	the pipe that the buffer belongs to
165 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,  in generic_pipe_buf_try_steal()  argument
185  * @pipe:	the pipe that the buffer belongs to
191  *	pipe into another.
193 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)  in generic_pipe_buf_get()  argument
201  * @pipe:	the pipe that the buffer belongs to
207 void generic_pipe_buf_release(struct pipe_inode_info *pipe,  in generic_pipe_buf_release()  argument
220 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
221 static inline bool pipe_readable(const struct pipe_inode_info *pipe)  in pipe_readable()  argument
223 	unsigned int head = READ_ONCE(pipe->head);  in pipe_readable()
224 	unsigned int tail = READ_ONCE(pipe->tail);  in pipe_readable()
225 	unsigned int writers = READ_ONCE(pipe->writers);  in pipe_readable()
235 	struct pipe_inode_info *pipe = filp->private_data;  in pipe_read()  local
244 	__pipe_lock(pipe);  in pipe_read()
247 	 * We only wake up writers if the pipe was full when we started  in pipe_read()
254 	was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);  in pipe_read()
257 		unsigned int head = smp_load_acquire(&pipe->head);  in pipe_read()
258 		unsigned int tail = pipe->tail;  in pipe_read()
259 		unsigned int mask = pipe->ring_size - 1;  in pipe_read()
262 		if (pipe->note_loss) {  in pipe_read()
281 			pipe->note_loss = false;  in pipe_read()
286 			struct pipe_buffer *buf = &pipe->bufs[tail & mask];  in pipe_read()
300 			error = pipe_buf_confirm(pipe, buf);  in pipe_read()
324 				pipe_buf_release(pipe, buf);  in pipe_read()
325 				spin_lock_irq(&pipe->rd_wait.lock);  in pipe_read()
328 					pipe->note_loss = true;  in pipe_read()
331 				pipe->tail = tail;  in pipe_read()
332 				spin_unlock_irq(&pipe->rd_wait.lock);  in pipe_read()
341 		if (!pipe->writers)  in pipe_read()
349 		__pipe_unlock(pipe);  in pipe_read()
355 		 * pipe buffer, and might have made space in the buffers  in pipe_read()
358 		 * You can't make zero-sized pipe buffers by doing an empty  in pipe_read()
365 		 * _very_ unlikely case that the pipe was full, but we got  in pipe_read()
369 			wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);  in pipe_read()
370 		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in pipe_read()
378 		if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)  in pipe_read()
381 		__pipe_lock(pipe);  in pipe_read()
382 		was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);  in pipe_read()
385 	if (pipe_empty(pipe->head, pipe->tail))  in pipe_read()
387 	__pipe_unlock(pipe);  in pipe_read()
390 		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);  in pipe_read()
392 		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);  in pipe_read()
393 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in pipe_read()
404 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
405 static inline bool pipe_writable(const struct pipe_inode_info *pipe)  in pipe_writable()  argument
407 	unsigned int head = READ_ONCE(pipe->head);  in pipe_writable()
408 	unsigned int tail = READ_ONCE(pipe->tail);  in pipe_writable()
409 	unsigned int max_usage = READ_ONCE(pipe->max_usage);  in pipe_writable()
412 		!READ_ONCE(pipe->readers);  in pipe_writable()
419 	struct pipe_inode_info *pipe = filp->private_data;  in pipe_write()  local
431 	__pipe_lock(pipe);  in pipe_write()
433 	if (!pipe->readers) {  in pipe_write()
440 	if (pipe->watch_queue) {  in pipe_write()
454 	head = pipe->head;  in pipe_write()
455 	was_empty = pipe_empty(head, pipe->tail);  in pipe_write()
458 		unsigned int mask = pipe->ring_size - 1;  in pipe_write()
459 		struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];  in pipe_write()
464 			ret = pipe_buf_confirm(pipe, buf);  in pipe_write()
481 		if (!pipe->readers) {  in pipe_write()
488 		head = pipe->head;  in pipe_write()
489 		if (!pipe_full(head, pipe->tail, pipe->max_usage)) {  in pipe_write()
490 			unsigned int mask = pipe->ring_size - 1;  in pipe_write()
491 			struct pipe_buffer *buf = &pipe->bufs[head & mask];  in pipe_write()
492 			struct page *page = pipe->tmp_page;  in pipe_write()
501 				pipe->tmp_page = page;  in pipe_write()
509 			spin_lock_irq(&pipe->rd_wait.lock);  in pipe_write()
511 			head = pipe->head;  in pipe_write()
512 			if (pipe_full(head, pipe->tail, pipe->max_usage)) {  in pipe_write()
513 				spin_unlock_irq(&pipe->rd_wait.lock);  in pipe_write()
517 			pipe->head = head + 1;  in pipe_write()
518 			spin_unlock_irq(&pipe->rd_wait.lock);  in pipe_write()
521 			buf = &pipe->bufs[head & mask];  in pipe_write()
530 			pipe->tmp_page = NULL;  in pipe_write()
546 		if (!pipe_full(head, pipe->tail, pipe->max_usage))  in pipe_write()
562 		 * We're going to release the pipe lock and wait for more  in pipe_write()
564 		 * after waiting we need to re-check whether the pipe  in pipe_write()
567 		__pipe_unlock(pipe);  in pipe_write()
569 			wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);  in pipe_write()
570 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in pipe_write()
571 		wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));  in pipe_write()
572 		__pipe_lock(pipe);  in pipe_write()
573 		was_empty = pipe_empty(pipe->head, pipe->tail);  in pipe_write()
577 	if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))  in pipe_write()
579 	__pipe_unlock(pipe);  in pipe_write()
590 	 * Epoll nonsensically wants a wakeup whether the pipe  in pipe_write()
593 	if (was_empty || pipe->poll_usage)  in pipe_write()
594 		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);  in pipe_write()
595 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in pipe_write()
597 		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);  in pipe_write()
609 	struct pipe_inode_info *pipe = filp->private_data;  in pipe_ioctl()  local
614 		__pipe_lock(pipe);  in pipe_ioctl()
616 		head = pipe->head;  in pipe_ioctl()
617 		tail = pipe->tail;  in pipe_ioctl()
618 		mask = pipe->ring_size - 1;  in pipe_ioctl()
621 			count += pipe->bufs[tail & mask].len;  in pipe_ioctl()
624 		__pipe_unlock(pipe);  in pipe_ioctl()
631 		__pipe_lock(pipe);  in pipe_ioctl()
632 		ret = watch_queue_set_size(pipe, arg);  in pipe_ioctl()
633 		__pipe_unlock(pipe);  in pipe_ioctl()
639 			pipe, (struct watch_notification_filter __user *)arg);  in pipe_ioctl()
652 	struct pipe_inode_info *pipe = filp->private_data;  in pipe_poll()  local
656 	WRITE_ONCE(pipe->poll_usage, true);  in pipe_poll()
659 	 * Reading pipe state only -- no need for acquiring the semaphore.  in pipe_poll()
665 		poll_wait(filp, &pipe->rd_wait, wait);  in pipe_poll()
667 		poll_wait(filp, &pipe->wr_wait, wait);  in pipe_poll()
674 	head = READ_ONCE(pipe->head);  in pipe_poll()
675 	tail = READ_ONCE(pipe->tail);  in pipe_poll()
681 		if (!pipe->writers && filp->f_version != pipe->w_counter)  in pipe_poll()
686 		if (!pipe_full(head, tail, pipe->max_usage))  in pipe_poll()
692 		if (!pipe->readers)  in pipe_poll()
699 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)  in put_pipe_info()  argument
704 	if (!--pipe->files) {  in put_pipe_info()
711 		free_pipe_info(pipe);  in put_pipe_info()
717 	struct pipe_inode_info *pipe = file->private_data;  in pipe_release()  local
719 	__pipe_lock(pipe);  in pipe_release()
721 		pipe->readers--;  in pipe_release()
723 		pipe->writers--;  in pipe_release()
726 	if (!pipe->readers != !pipe->writers) {  in pipe_release()
727 		wake_up_interruptible_all(&pipe->rd_wait);  in pipe_release()
728 		wake_up_interruptible_all(&pipe->wr_wait);  in pipe_release()
729 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in pipe_release()
730 		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in pipe_release()
732 	__pipe_unlock(pipe);  in pipe_release()
734 	put_pipe_info(inode, pipe);  in pipe_release()
741 	struct pipe_inode_info *pipe = filp->private_data;  in pipe_fasync()  local
744 	__pipe_lock(pipe);  in pipe_fasync()
746 		retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);  in pipe_fasync()
748 		retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);  in pipe_fasync()
751 			fasync_helper(-1, filp, 0, &pipe->fasync_readers);  in pipe_fasync()
753 	__pipe_unlock(pipe);  in pipe_fasync()
784 	struct pipe_inode_info *pipe;  in alloc_pipe_info()  local
790 	pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);  in alloc_pipe_info()
791 	if (pipe == NULL)  in alloc_pipe_info()
807 	pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),  in alloc_pipe_info()
810 	if (pipe->bufs) {  in alloc_pipe_info()
811 		init_waitqueue_head(&pipe->rd_wait);  in alloc_pipe_info()
812 		init_waitqueue_head(&pipe->wr_wait);  in alloc_pipe_info()
813 		pipe->r_counter = pipe->w_counter = 1;  in alloc_pipe_info()
814 		pipe->max_usage = pipe_bufs;  in alloc_pipe_info()
815 		pipe->ring_size = pipe_bufs;  in alloc_pipe_info()
816 		pipe->nr_accounted = pipe_bufs;  in alloc_pipe_info()
817 		pipe->user = user;  in alloc_pipe_info()
818 		mutex_init(&pipe->mutex);  in alloc_pipe_info()
819 		return pipe;  in alloc_pipe_info()
824 	kfree(pipe);  in alloc_pipe_info()
830 void free_pipe_info(struct pipe_inode_info *pipe)  in free_pipe_info()  argument
835 	if (pipe->watch_queue)  in free_pipe_info()
836 		watch_queue_clear(pipe->watch_queue);  in free_pipe_info()
839 	(void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);  in free_pipe_info()
840 	free_uid(pipe->user);  in free_pipe_info()
841 	for (i = 0; i < pipe->ring_size; i++) {  in free_pipe_info()
842 		struct pipe_buffer *buf = pipe->bufs + i;  in free_pipe_info()
844 			pipe_buf_release(pipe, buf);  in free_pipe_info()
847 	if (pipe->watch_queue)  in free_pipe_info()
848 		put_watch_queue(pipe->watch_queue);  in free_pipe_info()
850 	if (pipe->tmp_page)  in free_pipe_info()
851 		__free_page(pipe->tmp_page);  in free_pipe_info()
852 	kfree(pipe->bufs);  in free_pipe_info()
853 	kfree(pipe);  in free_pipe_info()
863 	return dynamic_dname(buffer, buflen, "pipe:[%lu]",  in pipefs_dname()
874 	struct pipe_inode_info *pipe;  in get_pipe_inode()  local
881 	pipe = alloc_pipe_info();  in get_pipe_inode()
882 	if (!pipe)  in get_pipe_inode()
885 	inode->i_pipe = pipe;  in get_pipe_inode()
886 	pipe->files = 2;  in get_pipe_inode()
887 	pipe->readers = pipe->writers = 1;  in get_pipe_inode()
1002  * a pipe. It's not the way Unix traditionally does this, though.
1031 SYSCALL_DEFINE1(pipe, int __user *, fildes)  in SYSCALL_DEFINE1()  argument
1037  * This is the stupid "wait for pipe to be readable or writable"
1045 void pipe_wait_readable(struct pipe_inode_info *pipe)  in pipe_wait_readable()  argument
1047 	pipe_unlock(pipe);  in pipe_wait_readable()
1048 	wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));  in pipe_wait_readable()
1049 	pipe_lock(pipe);  in pipe_wait_readable()
1052 void pipe_wait_writable(struct pipe_inode_info *pipe)  in pipe_wait_writable()  argument
1054 	pipe_unlock(pipe);  in pipe_wait_writable()
1055 	wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));  in pipe_wait_writable()
1056 	pipe_lock(pipe);  in pipe_wait_writable()
1061  * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1066  * because of the pipe lock, we can check the condition before being on
1069  * We use the 'rd_wait' waitqueue for pipe partner waiting.
1071 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)  in wait_for_partner()  argument
1077 		prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);  in wait_for_partner()
1078 		pipe_unlock(pipe);  in wait_for_partner()
1080 		finish_wait(&pipe->rd_wait, &rdwait);  in wait_for_partner()
1081 		pipe_lock(pipe);  in wait_for_partner()
1088 static void wake_up_partner(struct pipe_inode_info *pipe)  in wake_up_partner()  argument
1090 	wake_up_interruptible_all(&pipe->rd_wait);  in wake_up_partner()
1095 	struct pipe_inode_info *pipe;  in fifo_open()  local
1103 		pipe = inode->i_pipe;  in fifo_open()
1104 		pipe->files++;  in fifo_open()
1108 		pipe = alloc_pipe_info();  in fifo_open()
1109 		if (!pipe)  in fifo_open()
1111 		pipe->files = 1;  in fifo_open()
1116 			free_pipe_info(pipe);  in fifo_open()
1117 			pipe = inode->i_pipe;  in fifo_open()
1119 			inode->i_pipe = pipe;  in fifo_open()
1123 	filp->private_data = pipe;  in fifo_open()
1124 	/* OK, we have a pipe and it's pinned down */  in fifo_open()
1126 	__pipe_lock(pipe);  in fifo_open()
1138 		pipe->r_counter++;  in fifo_open()
1139 		if (pipe->readers++ == 0)  in fifo_open()
1140 			wake_up_partner(pipe);  in fifo_open()
1142 		if (!is_pipe && !pipe->writers) {  in fifo_open()
1146 				filp->f_version = pipe->w_counter;  in fifo_open()
1148 				if (wait_for_partner(pipe, &pipe->w_counter))  in fifo_open()
1161 		if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)  in fifo_open()
1164 		pipe->w_counter++;  in fifo_open()
1165 		if (!pipe->writers++)  in fifo_open()
1166 			wake_up_partner(pipe);  in fifo_open()
1168 		if (!is_pipe && !pipe->readers) {  in fifo_open()
1169 			if (wait_for_partner(pipe, &pipe->r_counter))  in fifo_open()
1182 		pipe->readers++;  in fifo_open()
1183 		pipe->writers++;  in fifo_open()
1184 		pipe->r_counter++;  in fifo_open()
1185 		pipe->w_counter++;  in fifo_open()
1186 		if (pipe->readers == 1 || pipe->writers == 1)  in fifo_open()
1187 			wake_up_partner(pipe);  in fifo_open()
1196 	__pipe_unlock(pipe);  in fifo_open()
1200 	if (!--pipe->readers)  in fifo_open()
1201 		wake_up_interruptible(&pipe->wr_wait);  in fifo_open()
1206 	if (!--pipe->writers)  in fifo_open()
1207 		wake_up_interruptible_all(&pipe->rd_wait);  in fifo_open()
1212 	__pipe_unlock(pipe);  in fifo_open()
1214 	put_pipe_info(inode, pipe);  in fifo_open()
1231  * Currently we rely on the pipe array holding a power-of-2 number
1239 	/* Minimum pipe size, as required by POSIX */  in round_pipe_size()
1247  * Resize the pipe ring to a number of slots.
1249  * Note the pipe can be reduced in capacity, but only if the current
1253 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)  in pipe_resize_ring()  argument
1263 	spin_lock_irq(&pipe->rd_wait.lock);  in pipe_resize_ring()
1264 	mask = pipe->ring_size - 1;  in pipe_resize_ring()
1265 	head = pipe->head;  in pipe_resize_ring()
1266 	tail = pipe->tail;  in pipe_resize_ring()
1270 		spin_unlock_irq(&pipe->rd_wait.lock);  in pipe_resize_ring()
1276 	 * The pipe array wraps around, so just start the new one at zero  in pipe_resize_ring()
1283 			memcpy(bufs, pipe->bufs + t,  in pipe_resize_ring()
1286 			unsigned int tsize = pipe->ring_size - t;  in pipe_resize_ring()
1288 				memcpy(bufs + tsize, pipe->bufs,  in pipe_resize_ring()
1290 			memcpy(bufs, pipe->bufs + t,  in pipe_resize_ring()
1298 	kfree(pipe->bufs);  in pipe_resize_ring()
1299 	pipe->bufs = bufs;  in pipe_resize_ring()
1300 	pipe->ring_size = nr_slots;  in pipe_resize_ring()
1301 	if (pipe->max_usage > nr_slots)  in pipe_resize_ring()
1302 		pipe->max_usage = nr_slots;  in pipe_resize_ring()
1303 	pipe->tail = tail;  in pipe_resize_ring()
1304 	pipe->head = head;  in pipe_resize_ring()
1306 	spin_unlock_irq(&pipe->rd_wait.lock);  in pipe_resize_ring()
1309 	wake_up_interruptible(&pipe->wr_wait);  in pipe_resize_ring()
1314  * Allocate a new array of pipe buffers and copy the info over. Returns the
1315  * pipe size if successful, or return -ERROR on error.
1317 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)  in pipe_set_size()  argument
1324 	if (pipe->watch_queue)  in pipe_set_size()
1335 	 * If trying to increase the pipe capacity, check that an  in pipe_set_size()
1338 	 * Decreasing the pipe capacity is always permitted, even  in pipe_set_size()
1341 	if (nr_slots > pipe->max_usage &&  in pipe_set_size()
1345 	user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);  in pipe_set_size()
1347 	if (nr_slots > pipe->max_usage &&  in pipe_set_size()
1355 	ret = pipe_resize_ring(pipe, nr_slots);  in pipe_set_size()
1359 	pipe->max_usage = nr_slots;  in pipe_set_size()
1360 	pipe->nr_accounted = nr_slots;  in pipe_set_size()
1361 	return pipe->max_usage * PAGE_SIZE;  in pipe_set_size()
1364 	(void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);  in pipe_set_size()
1370  * not enough to verify that this is a pipe.
1374 	struct pipe_inode_info *pipe = file->private_data;  in get_pipe_info()  local
1376 	if (file->f_op != &pipefifo_fops || !pipe)  in get_pipe_info()
1379 	if (for_splice && pipe->watch_queue)  in get_pipe_info()
1382 	return pipe;  in get_pipe_info()
1387 	struct pipe_inode_info *pipe;  in pipe_fcntl()  local
1390 	pipe = get_pipe_info(file, false);  in pipe_fcntl()
1391 	if (!pipe)  in pipe_fcntl()
1394 	__pipe_lock(pipe);  in pipe_fcntl()
1398 		ret = pipe_set_size(pipe, arg);  in pipe_fcntl()
1401 		ret = pipe->max_usage * PAGE_SIZE;  in pipe_fcntl()
1408 	__pipe_unlock(pipe);  in pipe_fcntl()
1421  * d_name - pipe: will go nicely and kill the special-casing in procfs.
1470 		.procname	= "pipe-max-size",
1477 		.procname	= "pipe-user-pages-hard",
1484 		.procname	= "pipe-user-pages-soft",