Lines Matching +full:op +full:- +full:tee
1 // SPDX-License-Identifier: GPL-2.0-only
6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
49 struct page *page = buf->page; in page_cache_pipe_buf_try_steal()
77 buf->flags |= PIPE_BUF_FLAG_LRU; in page_cache_pipe_buf_try_steal()
94 put_page(buf->page); in page_cache_pipe_buf_release()
95 buf->flags &= ~PIPE_BUF_FLAG_LRU; in page_cache_pipe_buf_release()
105 struct page *page = buf->page; in page_cache_pipe_buf_confirm()
112 * Page got truncated/unhashed. This will cause a 0-byte in page_cache_pipe_buf_confirm()
115 if (!page->mapping) { in page_cache_pipe_buf_confirm()
116 err = -ENODATA; in page_cache_pipe_buf_confirm()
121 * Uh oh, read-error from disk. in page_cache_pipe_buf_confirm()
124 err = -EIO; in page_cache_pipe_buf_confirm()
150 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) in user_page_pipe_buf_try_steal()
153 buf->flags |= PIPE_BUF_FLAG_LRU; in user_page_pipe_buf_try_steal()
166 if (waitqueue_active(&pipe->rd_wait)) in wakeup_pipe_readers()
167 wake_up_interruptible(&pipe->rd_wait); in wakeup_pipe_readers()
168 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in wakeup_pipe_readers()
172 * splice_to_pipe - fill passed data into a pipe
185 unsigned int spd_pages = spd->nr_pages; in splice_to_pipe()
186 unsigned int tail = pipe->tail; in splice_to_pipe()
187 unsigned int head = pipe->head; in splice_to_pipe()
188 unsigned int mask = pipe->ring_size - 1; in splice_to_pipe()
194 if (unlikely(!pipe->readers)) { in splice_to_pipe()
196 ret = -EPIPE; in splice_to_pipe()
200 while (!pipe_full(head, tail, pipe->max_usage)) { in splice_to_pipe()
201 struct pipe_buffer *buf = &pipe->bufs[head & mask]; in splice_to_pipe()
203 buf->page = spd->pages[page_nr]; in splice_to_pipe()
204 buf->offset = spd->partial[page_nr].offset; in splice_to_pipe()
205 buf->len = spd->partial[page_nr].len; in splice_to_pipe()
206 buf->private = spd->partial[page_nr].private; in splice_to_pipe()
207 buf->ops = spd->ops; in splice_to_pipe()
208 buf->flags = 0; in splice_to_pipe()
211 pipe->head = head; in splice_to_pipe()
213 ret += buf->len; in splice_to_pipe()
215 if (!--spd->nr_pages) in splice_to_pipe()
220 ret = -EAGAIN; in splice_to_pipe()
224 spd->spd_release(spd, page_nr++); in splice_to_pipe()
232 unsigned int head = pipe->head; in add_to_pipe()
233 unsigned int tail = pipe->tail; in add_to_pipe()
234 unsigned int mask = pipe->ring_size - 1; in add_to_pipe()
237 if (unlikely(!pipe->readers)) { in add_to_pipe()
239 ret = -EPIPE; in add_to_pipe()
240 } else if (pipe_full(head, tail, pipe->max_usage)) { in add_to_pipe()
241 ret = -EAGAIN; in add_to_pipe()
243 pipe->bufs[head & mask] = *buf; in add_to_pipe()
244 pipe->head = head + 1; in add_to_pipe()
245 return buf->len; in add_to_pipe()
258 unsigned int max_usage = READ_ONCE(pipe->max_usage); in splice_grow_spd()
260 spd->nr_pages_max = max_usage; in splice_grow_spd()
264 spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL); in splice_grow_spd()
265 spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page), in splice_grow_spd()
268 if (spd->pages && spd->partial) in splice_grow_spd()
271 kfree(spd->pages); in splice_grow_spd()
272 kfree(spd->partial); in splice_grow_spd()
273 return -ENOMEM; in splice_grow_spd()
278 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) in splice_shrink_spd()
281 kfree(spd->pages); in splice_shrink_spd()
282 kfree(spd->partial); in splice_shrink_spd()
286 * generic_file_splice_read - splice data from file to a pipe
295 * used as long as it has more or less sane ->read_iter().
320 * callers of ->splice_read() expect -EAGAIN on in generic_file_splice_read()
321 * "can't put anything in there", rather than -EFAULT. in generic_file_splice_read()
323 if (ret == -EFAULT) in generic_file_splice_read()
324 ret = -EAGAIN; in generic_file_splice_read()
345 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
351 struct file *file = sd->u.file; in pipe_to_sendpage()
352 loff_t pos = sd->pos; in pipe_to_sendpage()
355 if (!likely(file->f_op->sendpage)) in pipe_to_sendpage()
356 return -EINVAL; in pipe_to_sendpage()
358 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; in pipe_to_sendpage()
360 if (sd->len < sd->total_len && in pipe_to_sendpage()
361 pipe_occupancy(pipe->head, pipe->tail) > 1) in pipe_to_sendpage()
364 return file->f_op->sendpage(file, buf->page, buf->offset, in pipe_to_sendpage()
365 sd->len, &pos, more); in pipe_to_sendpage()
371 if (waitqueue_active(&pipe->wr_wait)) in wakeup_pipe_writers()
372 wake_up_interruptible(&pipe->wr_wait); in wakeup_pipe_writers()
373 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in wakeup_pipe_writers()
377 * splice_from_pipe_feed - feed available data from a pipe to a file
386 * the pipe or if the requested number of bytes (@sd->total_len)
389 * number of bytes have been copied and -errno on error.
399 unsigned int head = pipe->head; in splice_from_pipe_feed()
400 unsigned int tail = pipe->tail; in splice_from_pipe_feed()
401 unsigned int mask = pipe->ring_size - 1; in splice_from_pipe_feed()
405 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in splice_from_pipe_feed()
407 sd->len = buf->len; in splice_from_pipe_feed()
408 if (sd->len > sd->total_len) in splice_from_pipe_feed()
409 sd->len = sd->total_len; in splice_from_pipe_feed()
413 if (ret == -ENODATA) in splice_from_pipe_feed()
422 buf->offset += ret; in splice_from_pipe_feed()
423 buf->len -= ret; in splice_from_pipe_feed()
425 sd->num_spliced += ret; in splice_from_pipe_feed()
426 sd->len -= ret; in splice_from_pipe_feed()
427 sd->pos += ret; in splice_from_pipe_feed()
428 sd->total_len -= ret; in splice_from_pipe_feed()
430 if (!buf->len) { in splice_from_pipe_feed()
433 pipe->tail = tail; in splice_from_pipe_feed()
434 if (pipe->files) in splice_from_pipe_feed()
435 sd->need_wakeup = true; in splice_from_pipe_feed()
438 if (!sd->total_len) in splice_from_pipe_feed()
448 unsigned int tail = pipe->tail; in eat_empty_buffer()
449 unsigned int mask = pipe->ring_size - 1; in eat_empty_buffer()
450 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in eat_empty_buffer()
452 if (unlikely(!buf->len)) { in eat_empty_buffer()
454 pipe->tail = tail+1; in eat_empty_buffer()
462 * splice_from_pipe_next - wait for some data to splice from
469 * or -errno if no more data needs to be spliced.
478 return -ERESTARTSYS; in splice_from_pipe_next()
481 while (pipe_empty(pipe->head, pipe->tail)) { in splice_from_pipe_next()
482 if (!pipe->writers) in splice_from_pipe_next()
485 if (sd->num_spliced) in splice_from_pipe_next()
488 if (sd->flags & SPLICE_F_NONBLOCK) in splice_from_pipe_next()
489 return -EAGAIN; in splice_from_pipe_next()
492 return -ERESTARTSYS; in splice_from_pipe_next()
494 if (sd->need_wakeup) { in splice_from_pipe_next()
496 sd->need_wakeup = false; in splice_from_pipe_next()
509 * splice_from_pipe_begin - start splicing from pipe
519 sd->num_spliced = 0; in splice_from_pipe_begin()
520 sd->need_wakeup = false; in splice_from_pipe_begin()
524 * splice_from_pipe_end - finish splicing from pipe
535 if (sd->need_wakeup) in splice_from_pipe_end()
540 * __splice_from_pipe - splice data from a pipe to given actor
566 return sd->num_spliced ? sd->num_spliced : ret; in __splice_from_pipe()
571 * splice_from_pipe - splice data from a pipe to a file
604 * iter_file_splice_write - splice data from a pipe to a file
614 * This one is ->write_iter-based.
627 int nbufs = pipe->max_usage; in iter_file_splice_write()
633 return -ENOMEM; in iter_file_splice_write()
648 if (unlikely(nbufs < pipe->max_usage)) { in iter_file_splice_write()
650 nbufs = pipe->max_usage; in iter_file_splice_write()
654 ret = -ENOMEM; in iter_file_splice_write()
659 head = pipe->head; in iter_file_splice_write()
660 tail = pipe->tail; in iter_file_splice_write()
661 mask = pipe->ring_size - 1; in iter_file_splice_write()
666 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in iter_file_splice_write()
667 size_t this_len = buf->len; in iter_file_splice_write()
669 /* zero-length bvecs are not supported, skip them */ in iter_file_splice_write()
676 if (ret == -ENODATA) in iter_file_splice_write()
681 array[n].bv_page = buf->page; in iter_file_splice_write()
683 array[n].bv_offset = buf->offset; in iter_file_splice_write()
684 left -= this_len; in iter_file_splice_write()
688 iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left); in iter_file_splice_write()
694 sd.total_len -= ret; in iter_file_splice_write()
698 tail = pipe->tail; in iter_file_splice_write()
700 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in iter_file_splice_write()
701 if (ret >= buf->len) { in iter_file_splice_write()
702 ret -= buf->len; in iter_file_splice_write()
703 buf->len = 0; in iter_file_splice_write()
706 pipe->tail = tail; in iter_file_splice_write()
707 if (pipe->files) in iter_file_splice_write()
710 buf->offset += ret; in iter_file_splice_write()
711 buf->len -= ret; in iter_file_splice_write()
731 * generic_splice_sendpage - splice data from a pipe to a socket
751 static int warn_unsupported(struct file *file, const char *op) in warn_unsupported() argument
755 op, file, current->pid, current->comm); in warn_unsupported()
756 return -EINVAL; in warn_unsupported()
765 if (unlikely(!out->f_op->splice_write)) in do_splice_from()
767 return out->f_op->splice_write(pipe, out, ppos, len, flags); in do_splice_from()
780 if (unlikely(!(in->f_mode & FMODE_READ))) in do_splice_to()
781 return -EBADF; in do_splice_to()
784 p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail); in do_splice_to()
794 if (unlikely(!in->f_op->splice_read)) in do_splice_to()
796 return in->f_op->splice_read(in, ppos, pipe, len, flags); in do_splice_to()
800 * splice_direct_to_actor - splices data directly between two non-pipes
823 * randomly drop data for eg socket -> socket splicing. Use the in splice_direct_to_actor()
826 i_mode = file_inode(in)->i_mode; in splice_direct_to_actor()
828 return -EINVAL; in splice_direct_to_actor()
834 pipe = current->splice_pipe; in splice_direct_to_actor()
838 return -ENOMEM; in splice_direct_to_actor()
845 pipe->readers = 1; in splice_direct_to_actor()
847 current->splice_pipe = pipe; in splice_direct_to_actor()
855 len = sd->total_len; in splice_direct_to_actor()
856 flags = sd->flags; in splice_direct_to_actor()
861 sd->flags &= ~SPLICE_F_NONBLOCK; in splice_direct_to_actor()
862 more = sd->flags & SPLICE_F_MORE; in splice_direct_to_actor()
864 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail)); in splice_direct_to_actor()
868 loff_t pos = sd->pos, prev_pos = pos; in splice_direct_to_actor()
875 sd->total_len = read_len; in splice_direct_to_actor()
883 sd->flags |= SPLICE_F_MORE; in splice_direct_to_actor()
885 sd->flags &= ~SPLICE_F_MORE; in splice_direct_to_actor()
893 sd->pos = prev_pos; in splice_direct_to_actor()
898 len -= ret; in splice_direct_to_actor()
899 sd->pos = pos; in splice_direct_to_actor()
902 sd->pos = prev_pos + ret; in splice_direct_to_actor()
908 pipe->tail = pipe->head = 0; in splice_direct_to_actor()
917 for (i = 0; i < pipe->ring_size; i++) { in splice_direct_to_actor()
918 struct pipe_buffer *buf = &pipe->bufs[i]; in splice_direct_to_actor()
920 if (buf->ops) in splice_direct_to_actor()
934 struct file *file = sd->u.file; in direct_splice_actor()
936 return do_splice_from(pipe, file, sd->opos, sd->total_len, in direct_splice_actor()
937 sd->flags); in direct_splice_actor()
941 * do_splice_direct - splices data directly between two files
953 * can splice directly through a process-private pipe.
969 if (unlikely(!(out->f_mode & FMODE_WRITE))) in do_splice_direct()
970 return -EBADF; in do_splice_direct()
972 if (unlikely(out->f_flags & O_APPEND)) in do_splice_direct()
973 return -EINVAL; in do_splice_direct()
990 if (unlikely(!pipe->readers)) { in wait_for_space()
992 return -EPIPE; in wait_for_space()
994 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) in wait_for_space()
997 return -EAGAIN; in wait_for_space()
999 return -ERESTARTSYS; in wait_for_space()
1036 if (unlikely(!(in->f_mode & FMODE_READ) || in do_splice()
1037 !(out->f_mode & FMODE_WRITE))) in do_splice()
1038 return -EBADF; in do_splice()
1045 return -ESPIPE; in do_splice()
1049 return -EINVAL; in do_splice()
1051 if ((in->f_flags | out->f_flags) & O_NONBLOCK) in do_splice()
1059 return -ESPIPE; in do_splice()
1061 if (!(out->f_mode & FMODE_PWRITE)) in do_splice()
1062 return -EINVAL; in do_splice()
1065 offset = out->f_pos; in do_splice()
1068 if (unlikely(out->f_flags & O_APPEND)) in do_splice()
1069 return -EINVAL; in do_splice()
1075 if (in->f_flags & O_NONBLOCK) in do_splice()
1083 out->f_pos = offset; in do_splice()
1092 return -ESPIPE; in do_splice()
1094 if (!(in->f_mode & FMODE_PREAD)) in do_splice()
1095 return -EINVAL; in do_splice()
1098 offset = in->f_pos; in do_splice()
1101 if (out->f_flags & O_NONBLOCK) in do_splice()
1106 in->f_pos = offset; in do_splice()
1113 return -EINVAL; in do_splice()
1129 return -ESPIPE; in __do_splice()
1131 return -ESPIPE; in __do_splice()
1135 return -EFAULT; in __do_splice()
1140 return -EFAULT; in __do_splice()
1149 return -EFAULT; in __do_splice()
1151 return -EFAULT; in __do_splice()
1181 int size = min_t(int, copied, PAGE_SIZE - start); in iter_to_pipe()
1196 copied -= size; in iter_to_pipe()
1205 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); in pipe_to_user()
1206 return n == sd->len ? n : -EFAULT; in pipe_to_user()
1225 return -EBADF; in vmsplice_to_user()
1238 * as splice-from-memory, where the regular splice is splice-from-file (or
1253 return -EBADF; in vmsplice_to_pipe()
1268 return -EBADF; in vmsplice_type()
1269 if (f.file->f_mode & FMODE_WRITE) { in vmsplice_type()
1271 } else if (f.file->f_mode & FMODE_READ) { in vmsplice_type()
1275 return -EBADF; in vmsplice_type()
1288 * - memcpy() the data internally, at which point we might as well just
1290 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1307 return -EINVAL; in SYSCALL_DEFINE4()
1343 return -EINVAL; in SYSCALL_DEFINE6()
1345 error = -EBADF; in SYSCALL_DEFINE6()
1371 if (!pipe_empty(pipe->head, pipe->tail)) in ipipe_prep()
1377 while (pipe_empty(pipe->head, pipe->tail)) { in ipipe_prep()
1379 ret = -ERESTARTSYS; in ipipe_prep()
1382 if (!pipe->writers) in ipipe_prep()
1385 ret = -EAGAIN; in ipipe_prep()
1407 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) in opipe_prep()
1413 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { in opipe_prep()
1414 if (!pipe->readers) { in opipe_prep()
1416 ret = -EPIPE; in opipe_prep()
1420 ret = -EAGAIN; in opipe_prep()
1424 ret = -ERESTARTSYS; in opipe_prep()
1461 * could deadlock (one doing tee from A -> B, the other from B -> A). in splice_pipe_to_pipe()
1465 i_tail = ipipe->tail; in splice_pipe_to_pipe()
1466 i_mask = ipipe->ring_size - 1; in splice_pipe_to_pipe()
1467 o_head = opipe->head; in splice_pipe_to_pipe()
1468 o_mask = opipe->ring_size - 1; in splice_pipe_to_pipe()
1473 if (!opipe->readers) { in splice_pipe_to_pipe()
1476 ret = -EPIPE; in splice_pipe_to_pipe()
1480 i_head = ipipe->head; in splice_pipe_to_pipe()
1481 o_tail = opipe->tail; in splice_pipe_to_pipe()
1483 if (pipe_empty(i_head, i_tail) && !ipipe->writers) in splice_pipe_to_pipe()
1491 pipe_full(o_head, o_tail, opipe->max_usage)) { in splice_pipe_to_pipe()
1497 ret = -EAGAIN; in splice_pipe_to_pipe()
1511 ibuf = &ipipe->bufs[i_tail & i_mask]; in splice_pipe_to_pipe()
1512 obuf = &opipe->bufs[o_head & o_mask]; in splice_pipe_to_pipe()
1514 if (len >= ibuf->len) { in splice_pipe_to_pipe()
1519 ibuf->ops = NULL; in splice_pipe_to_pipe()
1521 ipipe->tail = i_tail; in splice_pipe_to_pipe()
1523 o_len = obuf->len; in splice_pipe_to_pipe()
1525 opipe->head = o_head; in splice_pipe_to_pipe()
1533 ret = -EFAULT; in splice_pipe_to_pipe()
1542 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; in splice_pipe_to_pipe()
1543 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; in splice_pipe_to_pipe()
1545 obuf->len = len; in splice_pipe_to_pipe()
1546 ibuf->offset += len; in splice_pipe_to_pipe()
1547 ibuf->len -= len; in splice_pipe_to_pipe()
1550 opipe->head = o_head; in splice_pipe_to_pipe()
1553 len -= o_len; in splice_pipe_to_pipe()
1587 * could deadlock (one doing tee from A -> B, the other from B -> A). in link_pipe()
1591 i_tail = ipipe->tail; in link_pipe()
1592 i_mask = ipipe->ring_size - 1; in link_pipe()
1593 o_head = opipe->head; in link_pipe()
1594 o_mask = opipe->ring_size - 1; in link_pipe()
1597 if (!opipe->readers) { in link_pipe()
1600 ret = -EPIPE; in link_pipe()
1604 i_head = ipipe->head; in link_pipe()
1605 o_tail = opipe->tail; in link_pipe()
1612 pipe_full(o_head, o_tail, opipe->max_usage)) in link_pipe()
1615 ibuf = &ipipe->bufs[i_tail & i_mask]; in link_pipe()
1616 obuf = &opipe->bufs[o_head & o_mask]; in link_pipe()
1624 ret = -EFAULT; in link_pipe()
1634 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; in link_pipe()
1635 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; in link_pipe()
1637 if (obuf->len > len) in link_pipe()
1638 obuf->len = len; in link_pipe()
1639 ret += obuf->len; in link_pipe()
1640 len -= obuf->len; in link_pipe()
1643 opipe->head = o_head; in link_pipe()
1660 * This is a tee(1) implementation that works on pipes. It doesn't copy
1669 int ret = -EINVAL; in do_tee()
1671 if (unlikely(!(in->f_mode & FMODE_READ) || in do_tee()
1672 !(out->f_mode & FMODE_WRITE))) in do_tee()
1673 return -EBADF; in do_tee()
1680 if ((in->f_flags | out->f_flags) & O_NONBLOCK) in do_tee()
1698 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) in SYSCALL_DEFINE4() argument
1704 return -EINVAL; in SYSCALL_DEFINE4()
1709 error = -EBADF; in SYSCALL_DEFINE4()