46 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_try_steal()  argument
91 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_release()  argument
102 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_confirm()  argument
147 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,  in user_page_pipe_buf_try_steal()  argument
154 	return generic_pipe_buf_try_steal(pipe, buf);  in user_page_pipe_buf_try_steal()
163 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)  in wakeup_pipe_readers()  argument
166 	if (waitqueue_active(&pipe->rd_wait))  in wakeup_pipe_readers()
167 		wake_up_interruptible(&pipe->rd_wait);  in wakeup_pipe_readers()
168 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in wakeup_pipe_readers()
182 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,  in splice_to_pipe()  argument
186 	unsigned int tail = pipe->tail;  in splice_to_pipe()
187 	unsigned int head = pipe->head;  in splice_to_pipe()
188 	unsigned int mask = pipe->ring_size - 1;  in splice_to_pipe()
194 	if (unlikely(!pipe->readers)) {  in splice_to_pipe()
200 	while (!pipe_full(head, tail, pipe->max_usage)) {  in splice_to_pipe()
201 		struct pipe_buffer *buf = &pipe->bufs[head & mask];  in splice_to_pipe()
211 		pipe->head = head;  in splice_to_pipe()
230 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)  in add_to_pipe()  argument
232 	unsigned int head = pipe->head;  in add_to_pipe()
233 	unsigned int tail = pipe->tail;  in add_to_pipe()
234 	unsigned int mask = pipe->ring_size - 1;  in add_to_pipe()
237 	if (unlikely(!pipe->readers)) {  in add_to_pipe()
240 	} else if (pipe_full(head, tail, pipe->max_usage)) {  in add_to_pipe()
243 		pipe->bufs[head & mask] = *buf;  in add_to_pipe()
244 		pipe->head = head + 1;  in add_to_pipe()
247 	pipe_buf_release(pipe, buf);  in add_to_pipe()
256 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)  in splice_grow_spd()  argument
258 	unsigned int max_usage = READ_ONCE(pipe->max_usage);  in splice_grow_spd()
299 				 struct pipe_inode_info *pipe, size_t len,  in generic_file_splice_read()  argument
306 	iov_iter_pipe(&to, READ, pipe, len);  in generic_file_splice_read()
315 		pipe_discard_from(pipe, to.start_head);  in generic_file_splice_read()
345 static int pipe_to_sendpage(struct pipe_inode_info *pipe,  in pipe_to_sendpage()  argument
358 	    pipe_occupancy(pipe->head, pipe->tail) > 1)  in pipe_to_sendpage()
365 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)  in wakeup_pipe_writers()  argument
368 	if (waitqueue_active(&pipe->wr_wait))  in wakeup_pipe_writers()
369 		wake_up_interruptible(&pipe->wr_wait);  in wakeup_pipe_writers()
370 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in wakeup_pipe_writers()
393 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
396 	unsigned int head = pipe->head;  in splice_from_pipe_feed()
397 	unsigned int tail = pipe->tail;  in splice_from_pipe_feed()
398 	unsigned int mask = pipe->ring_size - 1;  in splice_from_pipe_feed()
402 		struct pipe_buffer *buf = &pipe->bufs[tail & mask];  in splice_from_pipe_feed()
408 		ret = pipe_buf_confirm(pipe, buf);  in splice_from_pipe_feed()
415 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
428 			pipe_buf_release(pipe, buf);  in splice_from_pipe_feed()
430 			pipe->tail = tail;  in splice_from_pipe_feed()
431 			if (pipe->files)  in splice_from_pipe_feed()
443 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)  in eat_empty_buffer()  argument
445 	unsigned int tail = pipe->tail;  in eat_empty_buffer()
446 	unsigned int mask = pipe->ring_size - 1;  in eat_empty_buffer()
447 	struct pipe_buffer *buf = &pipe->bufs[tail & mask];  in eat_empty_buffer()
450 		pipe_buf_release(pipe, buf);  in eat_empty_buffer()
451 		pipe->tail = tail+1;  in eat_empty_buffer()
468 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
478 	while (pipe_empty(pipe->head, pipe->tail)) {  in splice_from_pipe_next()
479 		if (!pipe->writers)  in splice_from_pipe_next()
492 			wakeup_pipe_writers(pipe);  in splice_from_pipe_next()
496 		pipe_wait_readable(pipe);  in splice_from_pipe_next()
499 	if (eat_empty_buffer(pipe))  in splice_from_pipe_next()
530 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
533 		wakeup_pipe_writers(pipe);  in splice_from_pipe_end()
549 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
557 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
559 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
561 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
581 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,  in splice_from_pipe()  argument
593 	pipe_lock(pipe);  in splice_from_pipe()
594 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
595 	pipe_unlock(pipe);  in splice_from_pipe()
615 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,  in iter_file_splice_write()  argument
624 	int nbufs = pipe->max_usage;  in iter_file_splice_write()
632 	pipe_lock(pipe);  in iter_file_splice_write()
641 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
645 		if (unlikely(nbufs < pipe->max_usage)) {  in iter_file_splice_write()
647 			nbufs = pipe->max_usage;  in iter_file_splice_write()
656 		head = pipe->head;  in iter_file_splice_write()
657 		tail = pipe->tail;  in iter_file_splice_write()
658 		mask = pipe->ring_size - 1;  in iter_file_splice_write()
663 			struct pipe_buffer *buf = &pipe->bufs[tail & mask];  in iter_file_splice_write()
671 			ret = pipe_buf_confirm(pipe, buf);  in iter_file_splice_write()
695 		tail = pipe->tail;  in iter_file_splice_write()
697 			struct pipe_buffer *buf = &pipe->bufs[tail & mask];  in iter_file_splice_write()
701 				pipe_buf_release(pipe, buf);  in iter_file_splice_write()
703 				pipe->tail = tail;  in iter_file_splice_write()
704 				if (pipe->files)  in iter_file_splice_write()
715 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
717 	pipe_unlock(pipe);  in iter_file_splice_write()
740 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,  in generic_splice_sendpage()  argument
743 	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);  in generic_splice_sendpage()
759 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,  in do_splice_from()  argument
764 	return out->f_op->splice_write(pipe, out, ppos, len, flags);  in do_splice_from()
771 			 struct pipe_inode_info *pipe, size_t len,  in do_splice_to()  argument
781 	p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);  in do_splice_to()
793 	return in->f_op->splice_read(in, ppos, pipe, len, flags);  in do_splice_to()
812 	struct pipe_inode_info *pipe;  in splice_direct_to_actor()  local
829 	pipe = current->splice_pipe;  in splice_direct_to_actor()
830 	if (unlikely(!pipe)) {  in splice_direct_to_actor()
831 		pipe = alloc_pipe_info();  in splice_direct_to_actor()
832 		if (!pipe)  in splice_direct_to_actor()
840 		pipe->readers = 1;  in splice_direct_to_actor()
842 		current->splice_pipe = pipe;  in splice_direct_to_actor()
859 	WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));  in splice_direct_to_actor()
865 		ret = do_splice_to(in, &pos, pipe, len, flags);  in splice_direct_to_actor()
886 		ret = actor(pipe, sd);  in splice_direct_to_actor()
903 	pipe->tail = pipe->head = 0;  in splice_direct_to_actor()
912 	for (i = 0; i < pipe->ring_size; i++) {  in splice_direct_to_actor()
913 		struct pipe_buffer *buf = &pipe->bufs[i];  in splice_direct_to_actor()
916 			pipe_buf_release(pipe, buf);  in splice_direct_to_actor()
926 static int direct_splice_actor(struct pipe_inode_info *pipe,  in direct_splice_actor()  argument
931 	return do_splice_from(pipe, file, sd->opos, sd->total_len,  in direct_splice_actor()
982 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)  in wait_for_space()  argument
985 		if (unlikely(!pipe->readers)) {  in wait_for_space()
989 		if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))  in wait_for_space()
995 		pipe_wait_writable(pipe);  in wait_for_space()
1152 			struct pipe_inode_info *pipe,  in iter_to_pipe()  argument
1181 			ret = add_to_pipe(pipe, &buf);  in iter_to_pipe()
1198 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in pipe_to_user()  argument
1212 	struct pipe_inode_info *pipe = get_pipe_info(file, true);  in vmsplice_to_user()  local
1220 	if (!pipe)  in vmsplice_to_user()
1224 		pipe_lock(pipe);  in vmsplice_to_user()
1225 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()
1226 		pipe_unlock(pipe);  in vmsplice_to_user()
1240 	struct pipe_inode_info *pipe;  in vmsplice_to_pipe()  local
1247 	pipe = get_pipe_info(file, true);  in vmsplice_to_pipe()
1248 	if (!pipe)  in vmsplice_to_pipe()
1251 	pipe_lock(pipe);  in vmsplice_to_pipe()
1252 	ret = wait_for_space(pipe, flags);  in vmsplice_to_pipe()
1254 		ret = iter_to_pipe(iter, pipe, buf_flag);  in vmsplice_to_pipe()
1255 	pipe_unlock(pipe);  in vmsplice_to_pipe()
1257 		wakeup_pipe_readers(pipe);  in vmsplice_to_pipe()
1359 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in ipipe_prep()  argument
1367 	if (!pipe_empty(pipe->head, pipe->tail))  in ipipe_prep()
1371 	pipe_lock(pipe);  in ipipe_prep()
1373 	while (pipe_empty(pipe->head, pipe->tail)) {  in ipipe_prep()
1378 		if (!pipe->writers)  in ipipe_prep()
1384 		pipe_wait_readable(pipe);  in ipipe_prep()
1387 	pipe_unlock(pipe);  in ipipe_prep()
1395 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in opipe_prep()  argument
1403 	if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))  in opipe_prep()
1407 	pipe_lock(pipe);  in opipe_prep()
1409 	while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {  in opipe_prep()
1410 		if (!pipe->readers) {  in opipe_prep()
1423 		pipe_wait_writable(pipe);  in opipe_prep()
1426 	pipe_unlock(pipe);  in opipe_prep()