47 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_steal()  argument
92 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_release()  argument
103 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_confirm()  argument
148 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,  in user_page_pipe_buf_steal()  argument
155 	return generic_pipe_buf_steal(pipe, buf);  in user_page_pipe_buf_steal()
165 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)  in wakeup_pipe_readers()  argument
168 	if (waitqueue_active(&pipe->wait))  in wakeup_pipe_readers()
169 		wake_up_interruptible(&pipe->wait);  in wakeup_pipe_readers()
170 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in wakeup_pipe_readers()
184 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,  in splice_to_pipe()  argument
193 	if (unlikely(!pipe->readers)) {  in splice_to_pipe()
199 	while (pipe->nrbufs < pipe->buffers) {  in splice_to_pipe()
200 		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);  in splice_to_pipe()
201 		struct pipe_buffer *buf = pipe->bufs + newbuf;  in splice_to_pipe()
210 		pipe->nrbufs++;  in splice_to_pipe()
229 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)  in add_to_pipe()  argument
233 	if (unlikely(!pipe->readers)) {  in add_to_pipe()
236 	} else if (pipe->nrbufs == pipe->buffers) {  in add_to_pipe()
239 		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);  in add_to_pipe()
240 		pipe->bufs[newbuf] = *buf;  in add_to_pipe()
241 		pipe->nrbufs++;  in add_to_pipe()
244 	pipe_buf_release(pipe, buf);  in add_to_pipe()
253 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)  in splice_grow_spd()  argument
255 	unsigned int buffers = READ_ONCE(pipe->buffers);  in splice_grow_spd()
296 				 struct pipe_inode_info *pipe, size_t len,  in generic_file_splice_read()  argument
303 	iov_iter_pipe(&to, READ, pipe, len);  in generic_file_splice_read()
334 int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,  in generic_pipe_buf_nosteal()  argument
366 				 struct pipe_inode_info *pipe, size_t len,  in default_file_splice_read()  argument
377 	if (pipe->nrbufs == pipe->buffers)  in default_file_splice_read()
386 	iov_iter_pipe(&to, READ, pipe, len + offset);  in default_file_splice_read()
403 	pipe->bufs[to.idx].offset = offset;  in default_file_splice_read()
404 	pipe->bufs[to.idx].len -= offset;  in default_file_splice_read()
434 static int pipe_to_sendpage(struct pipe_inode_info *pipe,  in pipe_to_sendpage()  argument
446 	if (sd->len < sd->total_len && pipe->nrbufs > 1)  in pipe_to_sendpage()
453 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)  in wakeup_pipe_writers()  argument
456 	if (waitqueue_active(&pipe->wait))  in wakeup_pipe_writers()
457 		wake_up_interruptible(&pipe->wait);  in wakeup_pipe_writers()
458 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in wakeup_pipe_writers()
481 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
486 	while (pipe->nrbufs) {  in splice_from_pipe_feed()
487 		struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;  in splice_from_pipe_feed()
493 		ret = pipe_buf_confirm(pipe, buf);  in splice_from_pipe_feed()
500 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
513 			pipe_buf_release(pipe, buf);  in splice_from_pipe_feed()
514 			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);  in splice_from_pipe_feed()
515 			pipe->nrbufs--;  in splice_from_pipe_feed()
516 			if (pipe->files)  in splice_from_pipe_feed()
537 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
546 	while (!pipe->nrbufs) {  in splice_from_pipe_next()
547 		if (!pipe->writers)  in splice_from_pipe_next()
550 		if (!pipe->waiting_writers && sd->num_spliced)  in splice_from_pipe_next()
560 			wakeup_pipe_writers(pipe);  in splice_from_pipe_next()
564 		pipe_wait(pipe);  in splice_from_pipe_next()
595 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
598 		wakeup_pipe_writers(pipe);  in splice_from_pipe_end()
614 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
622 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
624 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
626 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
646 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,  in splice_from_pipe()  argument
658 	pipe_lock(pipe);  in splice_from_pipe()
659 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
660 	pipe_unlock(pipe);  in splice_from_pipe()
680 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,  in iter_file_splice_write()  argument
689 	int nbufs = pipe->buffers;  in iter_file_splice_write()
697 	pipe_lock(pipe);  in iter_file_splice_write()
705 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
709 		if (unlikely(nbufs < pipe->buffers)) {  in iter_file_splice_write()
711 			nbufs = pipe->buffers;  in iter_file_splice_write()
722 		for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {  in iter_file_splice_write()
723 			struct pipe_buffer *buf = pipe->bufs + idx;  in iter_file_splice_write()
729 			if (idx == pipe->buffers - 1)  in iter_file_splice_write()
732 			ret = pipe_buf_confirm(pipe, buf);  in iter_file_splice_write()
756 			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;  in iter_file_splice_write()
760 				pipe_buf_release(pipe, buf);  in iter_file_splice_write()
761 				pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);  in iter_file_splice_write()
762 				pipe->nrbufs--;  in iter_file_splice_write()
763 				if (pipe->files)  in iter_file_splice_write()
774 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
776 	pipe_unlock(pipe);  in iter_file_splice_write()
786 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in write_pipe_buf()  argument
800 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,  in default_file_splice_write()  argument
806 	ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);  in default_file_splice_write()
826 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,  in generic_splice_sendpage()  argument
829 	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);  in generic_splice_sendpage()
837 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,  in do_splice_from()  argument
848 	return splice_write(pipe, out, ppos, len, flags);  in do_splice_from()
855 			 struct pipe_inode_info *pipe, size_t len,  in do_splice_to()  argument
877 	return splice_read(in, ppos, pipe, len, flags);  in do_splice_to()
896 	struct pipe_inode_info *pipe;  in splice_direct_to_actor()  local
915 	pipe = current->splice_pipe;  in splice_direct_to_actor()
916 	if (unlikely(!pipe)) {  in splice_direct_to_actor()
917 		pipe = alloc_pipe_info();  in splice_direct_to_actor()
918 		if (!pipe)  in splice_direct_to_actor()
926 		pipe->readers = 1;  in splice_direct_to_actor()
928 		current->splice_pipe = pipe;  in splice_direct_to_actor()
945 	WARN_ON_ONCE(pipe->nrbufs != 0);  in splice_direct_to_actor()
953 				 (pipe->buffers - pipe->nrbufs) << PAGE_SHIFT);  in splice_direct_to_actor()
954 		ret = do_splice_to(in, &pos, pipe, read_len, flags);  in splice_direct_to_actor()
975 		ret = actor(pipe, sd);  in splice_direct_to_actor()
992 	pipe->nrbufs = pipe->curbuf = 0;  in splice_direct_to_actor()
1001 	for (i = 0; i < pipe->buffers; i++) {  in splice_direct_to_actor()
1002 		struct pipe_buffer *buf = pipe->bufs + i;  in splice_direct_to_actor()
1005 			pipe_buf_release(pipe, buf);  in splice_direct_to_actor()
1015 static int direct_splice_actor(struct pipe_inode_info *pipe,  in direct_splice_actor()  argument
1020 	return do_splice_from(pipe, file, sd->opos, sd->total_len,  in direct_splice_actor()
1071 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)  in wait_for_space()  argument
1074 		if (unlikely(!pipe->readers)) {  in wait_for_space()
1078 		if (pipe->nrbufs != pipe->buffers)  in wait_for_space()
1084 		pipe->waiting_writers++;  in wait_for_space()
1085 		pipe_wait(pipe);  in wait_for_space()
1086 		pipe->waiting_writers--;  in wait_for_space()
1200 			struct pipe_inode_info *pipe,  in iter_to_pipe()  argument
1229 				ret = add_to_pipe(pipe, &buf);  in iter_to_pipe()
1245 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in pipe_to_user()  argument
1259 	struct pipe_inode_info *pipe = get_pipe_info(file);  in vmsplice_to_user()  local
1267 	if (!pipe)  in vmsplice_to_user()
1271 		pipe_lock(pipe);  in vmsplice_to_user()
1272 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()
1273 		pipe_unlock(pipe);  in vmsplice_to_user()
1287 	struct pipe_inode_info *pipe;  in vmsplice_to_pipe()  local
1294 	pipe = get_pipe_info(file);  in vmsplice_to_pipe()
1295 	if (!pipe)  in vmsplice_to_pipe()
1298 	pipe_lock(pipe);  in vmsplice_to_pipe()
1299 	ret = wait_for_space(pipe, flags);  in vmsplice_to_pipe()
1301 		ret = iter_to_pipe(iter, pipe, buf_flag);  in vmsplice_to_pipe()
1302 	pipe_unlock(pipe);  in vmsplice_to_pipe()
1304 		wakeup_pipe_readers(pipe);  in vmsplice_to_pipe()
1440 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in ipipe_prep()  argument
1448 	if (pipe->nrbufs)  in ipipe_prep()
1452 	pipe_lock(pipe);  in ipipe_prep()
1454 	while (!pipe->nrbufs) {  in ipipe_prep()
1459 		if (!pipe->writers)  in ipipe_prep()
1461 		if (!pipe->waiting_writers) {  in ipipe_prep()
1467 		pipe_wait(pipe);  in ipipe_prep()
1470 	pipe_unlock(pipe);  in ipipe_prep()
1478 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in opipe_prep()  argument
1486 	if (pipe->nrbufs < pipe->buffers)  in opipe_prep()
1490 	pipe_lock(pipe);  in opipe_prep()
1492 	while (pipe->nrbufs >= pipe->buffers) {  in opipe_prep()
1493 		if (!pipe->readers) {  in opipe_prep()
1506 		pipe->waiting_writers++;  in opipe_prep()
1507 		pipe_wait(pipe);  in opipe_prep()
1508 		pipe->waiting_writers--;  in opipe_prep()
1511 	pipe_unlock(pipe);  in opipe_prep()