Lines Matching full:pipe
5 * This is the "extended pipe" functionality, where a pipe is used as
6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
10 * that transfers data buffers to or from a pipe buffer.
41 * Attempt to steal a page from a pipe buffer. This should perhaps go into
46 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe, in page_cache_pipe_buf_try_steal() argument
91 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, in page_cache_pipe_buf_release() argument
102 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, in page_cache_pipe_buf_confirm() argument
147 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe, in user_page_pipe_buf_try_steal() argument
154 return generic_pipe_buf_try_steal(pipe, buf); in user_page_pipe_buf_try_steal()
163 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) in wakeup_pipe_readers() argument
166 if (waitqueue_active(&pipe->rd_wait)) in wakeup_pipe_readers()
167 wake_up_interruptible(&pipe->rd_wait); in wakeup_pipe_readers()
168 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in wakeup_pipe_readers()
172 * splice_to_pipe - fill passed data into a pipe
173 * @pipe: pipe to fill
179 * function will link that data to the pipe.
182 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, in splice_to_pipe() argument
186 unsigned int tail = pipe->tail; in splice_to_pipe()
187 unsigned int head = pipe->head; in splice_to_pipe()
188 unsigned int mask = pipe->ring_size - 1; in splice_to_pipe()
194 if (unlikely(!pipe->readers)) { in splice_to_pipe()
200 while (!pipe_full(head, tail, pipe->max_usage)) { in splice_to_pipe()
201 struct pipe_buffer *buf = &pipe->bufs[head & mask]; in splice_to_pipe()
211 pipe->head = head; in splice_to_pipe()
230 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in add_to_pipe() argument
232 unsigned int head = pipe->head; in add_to_pipe()
233 unsigned int tail = pipe->tail; in add_to_pipe()
234 unsigned int mask = pipe->ring_size - 1; in add_to_pipe()
237 if (unlikely(!pipe->readers)) { in add_to_pipe()
240 } else if (pipe_full(head, tail, pipe->max_usage)) { in add_to_pipe()
243 pipe->bufs[head & mask] = *buf; in add_to_pipe()
244 pipe->head = head + 1; in add_to_pipe()
247 pipe_buf_release(pipe, buf); in add_to_pipe()
256 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) in splice_grow_spd() argument
258 unsigned int max_usage = READ_ONCE(pipe->max_usage); in splice_grow_spd()
286 * generic_file_splice_read - splice data from file to a pipe
289 * @pipe: pipe to splice to
294 * Will read pages from given file and fill them into a pipe. Can be
299 struct pipe_inode_info *pipe, size_t len, in generic_file_splice_read() argument
307 iov_iter_pipe(&to, READ, pipe, len); in generic_file_splice_read()
337 /* Pipe buffer operations for a socket and similar. */
348 static int pipe_to_sendpage(struct pipe_inode_info *pipe, in pipe_to_sendpage() argument
361 pipe_occupancy(pipe->head, pipe->tail) > 1) in pipe_to_sendpage()
368 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) in wakeup_pipe_writers() argument
371 if (waitqueue_active(&pipe->wr_wait)) in wakeup_pipe_writers()
372 wake_up_interruptible(&pipe->wr_wait); in wakeup_pipe_writers()
373 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in wakeup_pipe_writers()
377 * splice_from_pipe_feed - feed available data from a pipe to a file
378 * @pipe: pipe to splice from
383 * This function loops over the pipe and calls @actor to do the
386 * the pipe or if the requested number of bytes (@sd->total_len)
388 * pipe needs to be filled with more data, zero if the required
393 * locking is required around copying the pipe buffers to the
396 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, in splice_from_pipe_feed() argument
399 unsigned int head = pipe->head; in splice_from_pipe_feed()
400 unsigned int tail = pipe->tail; in splice_from_pipe_feed()
401 unsigned int mask = pipe->ring_size - 1; in splice_from_pipe_feed()
405 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in splice_from_pipe_feed()
411 ret = pipe_buf_confirm(pipe, buf); in splice_from_pipe_feed()
418 ret = actor(pipe, buf, sd); in splice_from_pipe_feed()
431 pipe_buf_release(pipe, buf); in splice_from_pipe_feed()
433 pipe->tail = tail; in splice_from_pipe_feed()
434 if (pipe->files) in splice_from_pipe_feed()
445 /* We know we have a pipe buffer, but maybe it's empty? */
446 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe) in eat_empty_buffer() argument
448 unsigned int tail = pipe->tail; in eat_empty_buffer()
449 unsigned int mask = pipe->ring_size - 1; in eat_empty_buffer()
450 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in eat_empty_buffer()
453 pipe_buf_release(pipe, buf); in eat_empty_buffer()
454 pipe->tail = tail+1; in eat_empty_buffer()
463 * @pipe: pipe to splice from
468 * value (one) if pipe buffers are available. It will return zero
471 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) in splice_from_pipe_next() argument
481 while (pipe_empty(pipe->head, pipe->tail)) { in splice_from_pipe_next()
482 if (!pipe->writers) in splice_from_pipe_next()
495 wakeup_pipe_writers(pipe); in splice_from_pipe_next()
499 pipe_wait_readable(pipe); in splice_from_pipe_next()
502 if (eat_empty_buffer(pipe)) in splice_from_pipe_next()
509 * splice_from_pipe_begin - start splicing from pipe
524 * splice_from_pipe_end - finish splicing from pipe
525 * @pipe: pipe to splice from
529 * This function will wake up pipe writers if necessary. It should
533 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) in splice_from_pipe_end() argument
536 wakeup_pipe_writers(pipe); in splice_from_pipe_end()
540 * __splice_from_pipe - splice data from a pipe to given actor
541 * @pipe: pipe to splice from
546 * This function does little more than loop over the pipe and call
552 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, in __splice_from_pipe() argument
560 ret = splice_from_pipe_next(pipe, sd); in __splice_from_pipe()
562 ret = splice_from_pipe_feed(pipe, sd, actor); in __splice_from_pipe()
564 splice_from_pipe_end(pipe, sd); in __splice_from_pipe()
571 * splice_from_pipe - splice data from a pipe to a file
572 * @pipe: pipe to splice from
580 * See __splice_from_pipe. This function locks the pipe inode,
584 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, in splice_from_pipe() argument
596 pipe_lock(pipe); in splice_from_pipe()
597 ret = __splice_from_pipe(pipe, &sd, actor); in splice_from_pipe()
598 pipe_unlock(pipe); in splice_from_pipe()
604 * iter_file_splice_write - splice data from a pipe to a file
605 * @pipe: pipe info
613 * the given pipe inode to the given file.
618 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, in iter_file_splice_write() argument
627 int nbufs = pipe->max_usage; in iter_file_splice_write()
635 pipe_lock(pipe); in iter_file_splice_write()
644 ret = splice_from_pipe_next(pipe, &sd); in iter_file_splice_write()
648 if (unlikely(nbufs < pipe->max_usage)) { in iter_file_splice_write()
650 nbufs = pipe->max_usage; in iter_file_splice_write()
659 head = pipe->head; in iter_file_splice_write()
660 tail = pipe->tail; in iter_file_splice_write()
661 mask = pipe->ring_size - 1; in iter_file_splice_write()
666 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in iter_file_splice_write()
674 ret = pipe_buf_confirm(pipe, buf); in iter_file_splice_write()
698 tail = pipe->tail; in iter_file_splice_write()
700 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in iter_file_splice_write()
704 pipe_buf_release(pipe, buf); in iter_file_splice_write()
706 pipe->tail = tail; in iter_file_splice_write()
707 if (pipe->files) in iter_file_splice_write()
718 splice_from_pipe_end(pipe, &sd); in iter_file_splice_write()
720 pipe_unlock(pipe); in iter_file_splice_write()
731 * generic_splice_sendpage - splice data from a pipe to a socket
732 * @pipe: pipe to splice from
739 * Will send @len bytes from the pipe to a network socket. No data copying
743 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, in generic_splice_sendpage() argument
746 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); in generic_splice_sendpage()
760 * Attempt to initiate a splice from pipe to file.
762 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, in do_splice_from() argument
767 return out->f_op->splice_write(pipe, out, ppos, len, flags); in do_splice_from()
771 * Attempt to initiate a splice from a file to a pipe.
774 struct pipe_inode_info *pipe, size_t len, in do_splice_to() argument
783 /* Don't try to read more the pipe has space for. */ in do_splice_to()
784 p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail); in do_splice_to()
796 return in->f_op->splice_read(in, ppos, pipe, len, flags); in do_splice_to()
807 * points, without requiring an explicit pipe. Internally an allocated
808 * pipe is cached in the process, and reused during the lifetime of
815 struct pipe_inode_info *pipe; in splice_direct_to_actor() local
831 * neither in nor out is a pipe, setup an internal pipe attached to in splice_direct_to_actor()
834 pipe = current->splice_pipe; in splice_direct_to_actor()
835 if (unlikely(!pipe)) { in splice_direct_to_actor()
836 pipe = alloc_pipe_info(); in splice_direct_to_actor()
837 if (!pipe) in splice_direct_to_actor()
842 * out of the pipe right after the splice_to_pipe(). So set in splice_direct_to_actor()
845 pipe->readers = 1; in splice_direct_to_actor()
847 current->splice_pipe = pipe; in splice_direct_to_actor()
859 * Don't block on output, we have to drain the direct pipe. in splice_direct_to_actor()
864 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail)); in splice_direct_to_actor()
870 ret = do_splice_to(in, &pos, pipe, len, flags); in splice_direct_to_actor()
889 * could get stuck data in the internal pipe: in splice_direct_to_actor()
891 ret = actor(pipe, sd); in splice_direct_to_actor()
908 pipe->tail = pipe->head = 0; in splice_direct_to_actor()
915 * the pipe buffers in question: in splice_direct_to_actor()
917 for (i = 0; i < pipe->ring_size; i++) { in splice_direct_to_actor()
918 struct pipe_buffer *buf = &pipe->bufs[i]; in splice_direct_to_actor()
921 pipe_buf_release(pipe, buf); in splice_direct_to_actor()
931 static int direct_splice_actor(struct pipe_inode_info *pipe, in direct_splice_actor() argument
936 return do_splice_from(pipe, file, sd->opos, sd->total_len, in direct_splice_actor()
953 * can splice directly through a process-private pipe.
987 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) in wait_for_space() argument
990 if (unlikely(!pipe->readers)) { in wait_for_space()
994 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) in wait_for_space()
1000 pipe_wait_writable(pipe); in wait_for_space()
1157 struct pipe_inode_info *pipe, in iter_to_pipe() argument
1186 ret = add_to_pipe(pipe, &buf); in iter_to_pipe()
1202 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, in pipe_to_user() argument
1216 struct pipe_inode_info *pipe = get_pipe_info(file, true); in vmsplice_to_user() local
1224 if (!pipe) in vmsplice_to_user()
1228 pipe_lock(pipe); in vmsplice_to_user()
1229 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); in vmsplice_to_user()
1230 pipe_unlock(pipe); in vmsplice_to_user()
1237 * vmsplice splices a user address range into a pipe. It can be thought of
1239 * to file). In both cases the output is a pipe, naturally.
1244 struct pipe_inode_info *pipe; in vmsplice_to_pipe() local
1251 pipe = get_pipe_info(file, true); in vmsplice_to_pipe()
1252 if (!pipe) in vmsplice_to_pipe()
1255 pipe_lock(pipe); in vmsplice_to_pipe()
1256 ret = wait_for_space(pipe, flags); in vmsplice_to_pipe()
1258 ret = iter_to_pipe(iter, pipe, buf_flag); in vmsplice_to_pipe()
1259 pipe_unlock(pipe); in vmsplice_to_pipe()
1261 wakeup_pipe_readers(pipe); in vmsplice_to_pipe()
1282 * to a pipe, not the other way around. Splicing from user memory is a simple
1285 * a pipe. The reverse isn't quite as easy, though. There are two possible
1291 * has restriction limitations on both ends of the pipe).
1363 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) in ipipe_prep() argument
1368 * Check the pipe occupancy without the inode lock first. This function in ipipe_prep()
1371 if (!pipe_empty(pipe->head, pipe->tail)) in ipipe_prep()
1375 pipe_lock(pipe); in ipipe_prep()
1377 while (pipe_empty(pipe->head, pipe->tail)) { in ipipe_prep()
1382 if (!pipe->writers) in ipipe_prep()
1388 pipe_wait_readable(pipe); in ipipe_prep()
1391 pipe_unlock(pipe); in ipipe_prep()
1399 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) in opipe_prep() argument
1404 * Check pipe occupancy without the inode lock first. This function in opipe_prep()
1407 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) in opipe_prep()
1411 pipe_lock(pipe); in opipe_prep()
1413 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { in opipe_prep()
1414 if (!pipe->readers) { in opipe_prep()
1427 pipe_wait_writable(pipe); in opipe_prep()
1430 pipe_unlock(pipe); in opipe_prep()
1460 * grabbing by pipe info address. Otherwise two different processes in splice_pipe_to_pipe()
1488 * pipe is empty or the output pipe is full. in splice_pipe_to_pipe()
1528 * Get a reference to this pipe buffer, in splice_pipe_to_pipe()
1560 * If we put data in the output pipe, wakeup any potential readers. in splice_pipe_to_pipe()
1586 * grabbing by pipe info address. Otherwise two different processes in link_pipe()
1619 * Get a reference to this pipe buffer, in link_pipe()
1651 * If we put data in the output pipe, wakeup any potential readers. in link_pipe()
1661 * any data, it simply references the 'in' pages on the 'out' pipe.