Lines Matching refs:pipe
79 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass) in pipe_lock_nested() argument
81 if (pipe->files) in pipe_lock_nested()
82 mutex_lock_nested(&pipe->mutex, subclass); in pipe_lock_nested()
85 void pipe_lock(struct pipe_inode_info *pipe) in pipe_lock() argument
90 pipe_lock_nested(pipe, I_MUTEX_PARENT); in pipe_lock()
94 void pipe_unlock(struct pipe_inode_info *pipe) in pipe_unlock() argument
96 if (pipe->files) in pipe_unlock()
97 mutex_unlock(&pipe->mutex); in pipe_unlock()
101 static inline void __pipe_lock(struct pipe_inode_info *pipe) in __pipe_lock() argument
103 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT); in __pipe_lock()
106 static inline void __pipe_unlock(struct pipe_inode_info *pipe) in __pipe_unlock() argument
108 mutex_unlock(&pipe->mutex); in __pipe_unlock()
125 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, in anon_pipe_buf_release() argument
135 if (page_count(page) == 1 && !pipe->tmp_page) in anon_pipe_buf_release()
136 pipe->tmp_page = page; in anon_pipe_buf_release()
141 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe, in anon_pipe_buf_try_steal() argument
165 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe, in generic_pipe_buf_try_steal() argument
193 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in generic_pipe_buf_get() argument
207 void generic_pipe_buf_release(struct pipe_inode_info *pipe, in generic_pipe_buf_release() argument
221 static inline bool pipe_readable(const struct pipe_inode_info *pipe) in pipe_readable() argument
223 unsigned int head = READ_ONCE(pipe->head); in pipe_readable()
224 unsigned int tail = READ_ONCE(pipe->tail); in pipe_readable()
225 unsigned int writers = READ_ONCE(pipe->writers); in pipe_readable()
235 struct pipe_inode_info *pipe = filp->private_data; in pipe_read() local
244 __pipe_lock(pipe); in pipe_read()
254 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage); in pipe_read()
257 unsigned int head = smp_load_acquire(&pipe->head); in pipe_read()
258 unsigned int tail = pipe->tail; in pipe_read()
259 unsigned int mask = pipe->ring_size - 1; in pipe_read()
262 if (pipe->note_loss) { in pipe_read()
281 pipe->note_loss = false; in pipe_read()
286 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in pipe_read()
300 error = pipe_buf_confirm(pipe, buf); in pipe_read()
324 pipe_buf_release(pipe, buf); in pipe_read()
325 spin_lock_irq(&pipe->rd_wait.lock); in pipe_read()
328 pipe->note_loss = true; in pipe_read()
331 pipe->tail = tail; in pipe_read()
332 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_read()
341 if (!pipe->writers) in pipe_read()
349 __pipe_unlock(pipe); in pipe_read()
369 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_read()
370 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_read()
378 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0) in pipe_read()
381 __pipe_lock(pipe); in pipe_read()
382 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage); in pipe_read()
385 if (pipe_empty(pipe->head, pipe->tail)) in pipe_read()
387 __pipe_unlock(pipe); in pipe_read()
390 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_read()
392 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_read()
393 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_read()
405 static inline bool pipe_writable(const struct pipe_inode_info *pipe) in pipe_writable() argument
407 unsigned int head = READ_ONCE(pipe->head); in pipe_writable()
408 unsigned int tail = READ_ONCE(pipe->tail); in pipe_writable()
409 unsigned int max_usage = READ_ONCE(pipe->max_usage); in pipe_writable()
412 !READ_ONCE(pipe->readers); in pipe_writable()
419 struct pipe_inode_info *pipe = filp->private_data; in pipe_write() local
431 __pipe_lock(pipe); in pipe_write()
433 if (!pipe->readers) { in pipe_write()
440 if (pipe->watch_queue) { in pipe_write()
454 head = pipe->head; in pipe_write()
455 was_empty = pipe_empty(head, pipe->tail); in pipe_write()
458 unsigned int mask = pipe->ring_size - 1; in pipe_write()
459 struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask]; in pipe_write()
464 ret = pipe_buf_confirm(pipe, buf); in pipe_write()
481 if (!pipe->readers) { in pipe_write()
488 head = pipe->head; in pipe_write()
489 if (!pipe_full(head, pipe->tail, pipe->max_usage)) { in pipe_write()
490 unsigned int mask = pipe->ring_size - 1; in pipe_write()
491 struct pipe_buffer *buf = &pipe->bufs[head & mask]; in pipe_write()
492 struct page *page = pipe->tmp_page; in pipe_write()
501 pipe->tmp_page = page; in pipe_write()
509 spin_lock_irq(&pipe->rd_wait.lock); in pipe_write()
511 head = pipe->head; in pipe_write()
512 if (pipe_full(head, pipe->tail, pipe->max_usage)) { in pipe_write()
513 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_write()
517 pipe->head = head + 1; in pipe_write()
518 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_write()
521 buf = &pipe->bufs[head & mask]; in pipe_write()
530 pipe->tmp_page = NULL; in pipe_write()
546 if (!pipe_full(head, pipe->tail, pipe->max_usage)) in pipe_write()
567 __pipe_unlock(pipe); in pipe_write()
569 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_write()
570 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_write()
571 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe)); in pipe_write()
572 __pipe_lock(pipe); in pipe_write()
573 was_empty = pipe_empty(pipe->head, pipe->tail); in pipe_write()
577 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) in pipe_write()
579 __pipe_unlock(pipe); in pipe_write()
593 if (was_empty || pipe->poll_usage) in pipe_write()
594 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); in pipe_write()
595 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_write()
597 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); in pipe_write()
609 struct pipe_inode_info *pipe = filp->private_data; in pipe_ioctl() local
614 __pipe_lock(pipe); in pipe_ioctl()
616 head = pipe->head; in pipe_ioctl()
617 tail = pipe->tail; in pipe_ioctl()
618 mask = pipe->ring_size - 1; in pipe_ioctl()
621 count += pipe->bufs[tail & mask].len; in pipe_ioctl()
624 __pipe_unlock(pipe); in pipe_ioctl()
631 __pipe_lock(pipe); in pipe_ioctl()
632 ret = watch_queue_set_size(pipe, arg); in pipe_ioctl()
633 __pipe_unlock(pipe); in pipe_ioctl()
639 pipe, (struct watch_notification_filter __user *)arg); in pipe_ioctl()
652 struct pipe_inode_info *pipe = filp->private_data; in pipe_poll() local
656 WRITE_ONCE(pipe->poll_usage, true); in pipe_poll()
665 poll_wait(filp, &pipe->rd_wait, wait); in pipe_poll()
667 poll_wait(filp, &pipe->wr_wait, wait); in pipe_poll()
674 head = READ_ONCE(pipe->head); in pipe_poll()
675 tail = READ_ONCE(pipe->tail); in pipe_poll()
681 if (!pipe->writers && filp->f_version != pipe->w_counter) in pipe_poll()
686 if (!pipe_full(head, tail, pipe->max_usage)) in pipe_poll()
692 if (!pipe->readers) in pipe_poll()
699 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe) in put_pipe_info() argument
704 if (!--pipe->files) { in put_pipe_info()
711 free_pipe_info(pipe); in put_pipe_info()
717 struct pipe_inode_info *pipe = file->private_data; in pipe_release() local
719 __pipe_lock(pipe); in pipe_release()
721 pipe->readers--; in pipe_release()
723 pipe->writers--; in pipe_release()
726 if (!pipe->readers != !pipe->writers) { in pipe_release()
727 wake_up_interruptible_all(&pipe->rd_wait); in pipe_release()
728 wake_up_interruptible_all(&pipe->wr_wait); in pipe_release()
729 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in pipe_release()
730 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in pipe_release()
732 __pipe_unlock(pipe); in pipe_release()
734 put_pipe_info(inode, pipe); in pipe_release()
741 struct pipe_inode_info *pipe = filp->private_data; in pipe_fasync() local
744 __pipe_lock(pipe); in pipe_fasync()
746 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); in pipe_fasync()
748 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); in pipe_fasync()
751 fasync_helper(-1, filp, 0, &pipe->fasync_readers); in pipe_fasync()
753 __pipe_unlock(pipe); in pipe_fasync()
784 struct pipe_inode_info *pipe; in alloc_pipe_info() local
790 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT); in alloc_pipe_info()
791 if (pipe == NULL) in alloc_pipe_info()
807 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer), in alloc_pipe_info()
810 if (pipe->bufs) { in alloc_pipe_info()
811 init_waitqueue_head(&pipe->rd_wait); in alloc_pipe_info()
812 init_waitqueue_head(&pipe->wr_wait); in alloc_pipe_info()
813 pipe->r_counter = pipe->w_counter = 1; in alloc_pipe_info()
814 pipe->max_usage = pipe_bufs; in alloc_pipe_info()
815 pipe->ring_size = pipe_bufs; in alloc_pipe_info()
816 pipe->nr_accounted = pipe_bufs; in alloc_pipe_info()
817 pipe->user = user; in alloc_pipe_info()
818 mutex_init(&pipe->mutex); in alloc_pipe_info()
819 return pipe; in alloc_pipe_info()
824 kfree(pipe); in alloc_pipe_info()
830 void free_pipe_info(struct pipe_inode_info *pipe) in free_pipe_info() argument
835 if (pipe->watch_queue) in free_pipe_info()
836 watch_queue_clear(pipe->watch_queue); in free_pipe_info()
839 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); in free_pipe_info()
840 free_uid(pipe->user); in free_pipe_info()
841 for (i = 0; i < pipe->ring_size; i++) { in free_pipe_info()
842 struct pipe_buffer *buf = pipe->bufs + i; in free_pipe_info()
844 pipe_buf_release(pipe, buf); in free_pipe_info()
847 if (pipe->watch_queue) in free_pipe_info()
848 put_watch_queue(pipe->watch_queue); in free_pipe_info()
850 if (pipe->tmp_page) in free_pipe_info()
851 __free_page(pipe->tmp_page); in free_pipe_info()
852 kfree(pipe->bufs); in free_pipe_info()
853 kfree(pipe); in free_pipe_info()
874 struct pipe_inode_info *pipe; in get_pipe_inode() local
881 pipe = alloc_pipe_info(); in get_pipe_inode()
882 if (!pipe) in get_pipe_inode()
885 inode->i_pipe = pipe; in get_pipe_inode()
886 pipe->files = 2; in get_pipe_inode()
887 pipe->readers = pipe->writers = 1; in get_pipe_inode()
1031 SYSCALL_DEFINE1(pipe, int __user *, fildes) in SYSCALL_DEFINE1() argument
1045 void pipe_wait_readable(struct pipe_inode_info *pipe) in pipe_wait_readable() argument
1047 pipe_unlock(pipe); in pipe_wait_readable()
1048 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe)); in pipe_wait_readable()
1049 pipe_lock(pipe); in pipe_wait_readable()
1052 void pipe_wait_writable(struct pipe_inode_info *pipe) in pipe_wait_writable() argument
1054 pipe_unlock(pipe); in pipe_wait_writable()
1055 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe)); in pipe_wait_writable()
1056 pipe_lock(pipe); in pipe_wait_writable()
1071 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) in wait_for_partner() argument
1077 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE); in wait_for_partner()
1078 pipe_unlock(pipe); in wait_for_partner()
1080 finish_wait(&pipe->rd_wait, &rdwait); in wait_for_partner()
1081 pipe_lock(pipe); in wait_for_partner()
1088 static void wake_up_partner(struct pipe_inode_info *pipe) in wake_up_partner() argument
1090 wake_up_interruptible_all(&pipe->rd_wait); in wake_up_partner()
1095 struct pipe_inode_info *pipe; in fifo_open() local
1103 pipe = inode->i_pipe; in fifo_open()
1104 pipe->files++; in fifo_open()
1108 pipe = alloc_pipe_info(); in fifo_open()
1109 if (!pipe) in fifo_open()
1111 pipe->files = 1; in fifo_open()
1116 free_pipe_info(pipe); in fifo_open()
1117 pipe = inode->i_pipe; in fifo_open()
1119 inode->i_pipe = pipe; in fifo_open()
1123 filp->private_data = pipe; in fifo_open()
1126 __pipe_lock(pipe); in fifo_open()
1138 pipe->r_counter++; in fifo_open()
1139 if (pipe->readers++ == 0) in fifo_open()
1140 wake_up_partner(pipe); in fifo_open()
1142 if (!is_pipe && !pipe->writers) { in fifo_open()
1146 filp->f_version = pipe->w_counter; in fifo_open()
1148 if (wait_for_partner(pipe, &pipe->w_counter)) in fifo_open()
1161 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) in fifo_open()
1164 pipe->w_counter++; in fifo_open()
1165 if (!pipe->writers++) in fifo_open()
1166 wake_up_partner(pipe); in fifo_open()
1168 if (!is_pipe && !pipe->readers) { in fifo_open()
1169 if (wait_for_partner(pipe, &pipe->r_counter)) in fifo_open()
1182 pipe->readers++; in fifo_open()
1183 pipe->writers++; in fifo_open()
1184 pipe->r_counter++; in fifo_open()
1185 pipe->w_counter++; in fifo_open()
1186 if (pipe->readers == 1 || pipe->writers == 1) in fifo_open()
1187 wake_up_partner(pipe); in fifo_open()
1196 __pipe_unlock(pipe); in fifo_open()
1200 if (!--pipe->readers) in fifo_open()
1201 wake_up_interruptible(&pipe->wr_wait); in fifo_open()
1206 if (!--pipe->writers) in fifo_open()
1207 wake_up_interruptible_all(&pipe->rd_wait); in fifo_open()
1212 __pipe_unlock(pipe); in fifo_open()
1214 put_pipe_info(inode, pipe); in fifo_open()
1253 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) in pipe_resize_ring() argument
1263 spin_lock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1264 mask = pipe->ring_size - 1; in pipe_resize_ring()
1265 head = pipe->head; in pipe_resize_ring()
1266 tail = pipe->tail; in pipe_resize_ring()
1270 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1283 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1286 unsigned int tsize = pipe->ring_size - t; in pipe_resize_ring()
1288 memcpy(bufs + tsize, pipe->bufs, in pipe_resize_ring()
1290 memcpy(bufs, pipe->bufs + t, in pipe_resize_ring()
1298 kfree(pipe->bufs); in pipe_resize_ring()
1299 pipe->bufs = bufs; in pipe_resize_ring()
1300 pipe->ring_size = nr_slots; in pipe_resize_ring()
1301 if (pipe->max_usage > nr_slots) in pipe_resize_ring()
1302 pipe->max_usage = nr_slots; in pipe_resize_ring()
1303 pipe->tail = tail; in pipe_resize_ring()
1304 pipe->head = head; in pipe_resize_ring()
1306 spin_unlock_irq(&pipe->rd_wait.lock); in pipe_resize_ring()
1309 wake_up_interruptible(&pipe->wr_wait); in pipe_resize_ring()
1317 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg) in pipe_set_size() argument
1324 if (pipe->watch_queue) in pipe_set_size()
1341 if (nr_slots > pipe->max_usage && in pipe_set_size()
1345 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); in pipe_set_size()
1347 if (nr_slots > pipe->max_usage && in pipe_set_size()
1355 ret = pipe_resize_ring(pipe, nr_slots); in pipe_set_size()
1359 pipe->max_usage = nr_slots; in pipe_set_size()
1360 pipe->nr_accounted = nr_slots; in pipe_set_size()
1361 return pipe->max_usage * PAGE_SIZE; in pipe_set_size()
1364 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); in pipe_set_size()
1374 struct pipe_inode_info *pipe = file->private_data; in get_pipe_info() local
1376 if (file->f_op != &pipefifo_fops || !pipe) in get_pipe_info()
1379 if (for_splice && pipe->watch_queue) in get_pipe_info()
1382 return pipe; in get_pipe_info()
1387 struct pipe_inode_info *pipe; in pipe_fcntl() local
1390 pipe = get_pipe_info(file, false); in pipe_fcntl()
1391 if (!pipe) in pipe_fcntl()
1394 __pipe_lock(pipe); in pipe_fcntl()
1398 ret = pipe_set_size(pipe, arg); in pipe_fcntl()
1401 ret = pipe->max_usage * PAGE_SIZE; in pipe_fcntl()
1408 __pipe_unlock(pipe); in pipe_fcntl()