Lines Matching +full:poll +full:- +full:timeout +full:- +full:ms
1 // SPDX-License-Identifier: GPL-2.0-or-later
18 #include <linux/poll.h>
47 * 2) ep->mtx (mutex)
48 * 3) ep->lock (rwlock)
51 * We need a rwlock (ep->lock) because we manipulate objects
52 * from inside the poll callback, that might be triggered from
54 * So we can't sleep inside the poll callback and hence we need
58 * mutex (ep->mtx). It is acquired during the event transfer loop,
73 * It is necessary to acquire multiple "ep->mtx"es at once in the
76 * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
77 * before e2->mtx). Since we disallow cycles of epoll file
78 * descriptors, this ensures that the mutexes are well-ordered. In
82 * It is possible to drop the "ep->mtx" and to use the global
83 * mutex "epmutex" (together with "ep->lock") to have it working,
84 * but having "ep->mtx" will make the interface more scalable.
86 * normal operations the epoll private "ep->mtx" will guarantee
103 #define EP_UNACTIVE_PTR ((void *) -1L)
112 /* Wait structure used by the poll hooks */
148 * Works together "struct eventpoll"->ovflist in keeping the
156 /* List containing poll wait queues */
189 /* Wait queue used by file->poll() */
204 * holding ->lock.
221 /* used to track busy poll napi_id */
231 /* Wrapper struct used by poll queueing */
281 head = container_of(file->f_ep, struct epitems_head, epitems); in list_file()
282 if (!head->next) { in list_file()
283 head->next = tfile_check_list; in list_file()
291 struct hlist_node *p = rcu_dereference(hlist_first_rcu(&head->epitems)); in unlist_file()
294 spin_lock(&epi->ffd.file->f_lock); in unlist_file()
295 if (!hlist_empty(&head->epitems)) in unlist_file()
297 head->next = NULL; in unlist_file()
298 spin_unlock(&epi->ffd.file->f_lock); in unlist_file()
335 return f->f_op == &eventpoll_fops; in is_file_epoll()
342 ffd->file = file; in ep_set_ffd()
343 ffd->fd = fd; in ep_set_ffd()
350 return (p1->file > p2->file ? +1: in ep_cmp_ffd()
351 (p1->file < p2->file ? -1 : p1->fd - p2->fd)); in ep_cmp_ffd()
357 return !list_empty(&epi->rdllink); in ep_is_linked()
368 return container_of(p, struct eppoll_entry, wait)->base; in ep_item_from_wait()
372 * ep_events_available - Checks if ready events might be available.
381 return !list_empty_careful(&ep->rdllist) || in ep_events_available()
382 READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; in ep_events_available()
394 * Busy poll if globally on and supporting sockets found && no events,
401 unsigned int napi_id = READ_ONCE(ep->napi_id); in ep_busy_loop()
409 * Busy poll timed out. Drop NAPI ID for now, we can add in ep_busy_loop()
413 ep->napi_id = 0; in ep_busy_loop()
420 * Set epoll busy poll NAPI ID from sk.
432 sock = sock_from_file(epi->ffd.file); in ep_set_busy_poll_napi_id()
436 sk = sock->sk; in ep_set_busy_poll_napi_id()
440 napi_id = READ_ONCE(sk->sk_napi_id); in ep_set_busy_poll_napi_id()
441 ep = epi->ep; in ep_set_busy_poll_napi_id()
443 /* Non-NAPI IDs can be rejected in ep_set_busy_poll_napi_id()
447 if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id) in ep_set_busy_poll_napi_id()
450 /* record NAPI ID for use in next busy poll */ in ep_set_busy_poll_napi_id()
451 ep->napi_id = napi_id; in ep_set_busy_poll_napi_id()
480 * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a
484 * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()
502 * it might be natural to create a per-cpu nest count. However, since in ep_poll_safewake()
503 * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can in ep_poll_safewake()
504 * schedule() in the -rt kernel, the per-cpu variable are no longer in ep_poll_safewake()
516 if ((is_file_epoll(epi->ffd.file))) { in ep_poll_safewake()
517 ep_src = epi->ffd.file->private_data; in ep_poll_safewake()
518 nests = ep_src->nests; in ep_poll_safewake()
523 spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests); in ep_poll_safewake()
524 ep->nests = nests + 1; in ep_poll_safewake()
525 wake_up_locked_poll(&ep->poll_wait, EPOLLIN); in ep_poll_safewake()
526 ep->nests = 0; in ep_poll_safewake()
527 spin_unlock_irqrestore(&ep->poll_wait.lock, flags); in ep_poll_safewake()
534 wake_up_poll(&ep->poll_wait, EPOLLIN); in ep_poll_safewake()
545 * If it is cleared by POLLFREE, it should be rcu-safe. in ep_remove_wait_queue()
548 * we rely on whead->lock. in ep_remove_wait_queue()
550 whead = smp_load_acquire(&pwq->whead); in ep_remove_wait_queue()
552 remove_wait_queue(whead, &pwq->wait); in ep_remove_wait_queue()
557 * This function unregisters poll callbacks from the associated file
563 struct eppoll_entry **p = &epi->pwqlist; in ep_unregister_pollwait()
567 *p = pwq->next; in ep_unregister_pollwait()
573 /* call only when ep->mtx is held */
576 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
579 /* call only when ep->mtx is held */
590 return rcu_access_pointer(epi->ws) ? true : false; in ep_has_wakeup_source()
593 /* call when ep->mtx cannot be held (ep_poll_callback) */
599 ws = rcu_dereference(epi->ws); in ep_pm_stay_awake_rcu()
607 * ep->mutex needs to be held because we could be hit by
613 * Steal the ready list, and re-init the original one to the in ep_start_scan()
614 * empty list. Also, set ep->ovflist to NULL so that events in ep_start_scan()
616 * have the poll callback to queue directly on ep->rdllist, in ep_start_scan()
621 write_lock_irq(&ep->lock); in ep_start_scan()
622 list_splice_init(&ep->rdllist, txlist); in ep_start_scan()
623 WRITE_ONCE(ep->ovflist, NULL); in ep_start_scan()
624 write_unlock_irq(&ep->lock); in ep_start_scan()
632 write_lock_irq(&ep->lock); in ep_done_scan()
635 * other events might have been queued by the poll callback. in ep_done_scan()
636 * We re-insert them inside the main ready-list here. in ep_done_scan()
638 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; in ep_done_scan()
639 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { in ep_done_scan()
643 * queued into ->ovflist but the "txlist" might already in ep_done_scan()
648 * ->ovflist is LIFO, so we have to reverse it in order in ep_done_scan()
651 list_add(&epi->rdllink, &ep->rdllist); in ep_done_scan()
656 * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after in ep_done_scan()
658 * ep->rdllist. in ep_done_scan()
660 WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); in ep_done_scan()
663 * Quickly re-inject items left on "txlist". in ep_done_scan()
665 list_splice(txlist, &ep->rdllist); in ep_done_scan()
666 __pm_relax(ep->ws); in ep_done_scan()
668 if (!list_empty(&ep->rdllist)) { in ep_done_scan()
669 if (waitqueue_active(&ep->wq)) in ep_done_scan()
670 wake_up(&ep->wq); in ep_done_scan()
673 write_unlock_irq(&ep->lock); in ep_done_scan()
688 struct file *file = epi->ffd.file; in ep_remove()
695 * Removes poll wait queue hooks. in ep_remove()
700 spin_lock(&file->f_lock); in ep_remove()
702 head = file->f_ep; in ep_remove()
703 if (head->first == &epi->fllink && !epi->fllink.next) { in ep_remove()
704 file->f_ep = NULL; in ep_remove()
708 if (!smp_load_acquire(&v->next)) in ep_remove()
712 hlist_del_rcu(&epi->fllink); in ep_remove()
713 spin_unlock(&file->f_lock); in ep_remove()
716 rb_erase_cached(&epi->rbn, &ep->rbr); in ep_remove()
718 write_lock_irq(&ep->lock); in ep_remove()
720 list_del_init(&epi->rdllink); in ep_remove()
721 write_unlock_irq(&ep->lock); in ep_remove()
726 * field epi->rcu, since we are trying to minimize the size of in ep_remove()
728 * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make in ep_remove()
731 call_rcu(&epi->rcu, epi_rcu_free); in ep_remove()
733 percpu_counter_dec(&ep->user->epoll_watches); in ep_remove()
744 if (waitqueue_active(&ep->poll_wait)) in ep_free()
750 * We do not need to hold "ep->mtx" here because the epoll file in ep_free()
758 * Walks through the whole tree by unregistering poll callbacks. in ep_free()
760 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_free()
769 * point we are sure no poll callbacks will be lingering around, and also by in ep_free()
771 * us during this operation. So we can avoid the lock on "ep->lock". in ep_free()
772 * We do not need to lock ep->mtx, either, we only do it to prevent in ep_free()
775 mutex_lock(&ep->mtx); in ep_free()
776 while ((rbp = rb_first_cached(&ep->rbr)) != NULL) { in ep_free()
781 mutex_unlock(&ep->mtx); in ep_free()
784 mutex_destroy(&ep->mtx); in ep_free()
785 free_uid(ep->user); in ep_free()
786 wakeup_source_unregister(ep->ws); in ep_free()
792 struct eventpoll *ep = file->private_data; in ep_eventpoll_release()
804 struct eventpoll *ep = file->private_data; in __ep_eventpoll_poll()
812 /* Insert inside our poll wait queue */ in __ep_eventpoll_poll()
813 poll_wait(file, &ep->poll_wait, wait); in __ep_eventpoll_poll()
819 mutex_lock_nested(&ep->mtx, depth); in __ep_eventpoll_poll()
827 * Item has been dropped into the ready list by the poll in __ep_eventpoll_poll()
832 list_del_init(&epi->rdllink); in __ep_eventpoll_poll()
836 mutex_unlock(&ep->mtx); in __ep_eventpoll_poll()
842 * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
848 struct file *file = epi->ffd.file; in ep_item_poll()
851 pt->_key = epi->event.events; in ep_item_poll()
856 return res & epi->event.events; in ep_item_poll()
867 struct eventpoll *ep = f->private_data; in ep_show_fdinfo()
870 mutex_lock(&ep->mtx); in ep_show_fdinfo()
871 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_show_fdinfo()
873 struct inode *inode = file_inode(epi->ffd.file); in ep_show_fdinfo()
877 epi->ffd.fd, epi->event.events, in ep_show_fdinfo()
878 (long long)epi->event.data, in ep_show_fdinfo()
879 (long long)epi->ffd.file->f_pos, in ep_show_fdinfo()
880 inode->i_ino, inode->i_sb->s_dev); in ep_show_fdinfo()
884 mutex_unlock(&ep->mtx); in ep_show_fdinfo()
894 .poll = ep_eventpoll_poll,
910 * We don't want to get "file->f_lock" because it is not in eventpoll_release_file()
917 * "ep->mtx" after "epmutex" because ep_remove() requires it when called in eventpoll_release_file()
923 if (unlikely(!file->f_ep)) { in eventpoll_release_file()
927 hlist_for_each_entry_safe(epi, next, file->f_ep, fllink) { in eventpoll_release_file()
928 ep = epi->ep; in eventpoll_release_file()
929 mutex_lock_nested(&ep->mtx, 0); in eventpoll_release_file()
931 mutex_unlock(&ep->mtx); in eventpoll_release_file()
943 error = -ENOMEM; in ep_alloc()
948 mutex_init(&ep->mtx); in ep_alloc()
949 rwlock_init(&ep->lock); in ep_alloc()
950 init_waitqueue_head(&ep->wq); in ep_alloc()
951 init_waitqueue_head(&ep->poll_wait); in ep_alloc()
952 INIT_LIST_HEAD(&ep->rdllist); in ep_alloc()
953 ep->rbr = RB_ROOT_CACHED; in ep_alloc()
954 ep->ovflist = EP_UNACTIVE_PTR; in ep_alloc()
955 ep->user = user; in ep_alloc()
979 for (rbp = ep->rbr.rb_root.rb_node; rbp; ) { in ep_find()
981 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); in ep_find()
983 rbp = rbp->rb_right; in ep_find()
985 rbp = rbp->rb_left; in ep_find()
1001 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_find_tfd()
1003 if (epi->ffd.fd == tfd) { in ep_find_tfd()
1007 toff--; in ep_find_tfd()
1023 return ERR_PTR(-EINVAL); in get_epoll_tfile_raw_ptr()
1025 ep = file->private_data; in get_epoll_tfile_raw_ptr()
1027 mutex_lock(&ep->mtx); in get_epoll_tfile_raw_ptr()
1030 file_raw = epi->ffd.file; in get_epoll_tfile_raw_ptr()
1032 file_raw = ERR_PTR(-ENOENT); in get_epoll_tfile_raw_ptr()
1033 mutex_unlock(&ep->mtx); in get_epoll_tfile_raw_ptr()
1063 * This is simple 'new->next = head' operation, but cmpxchg() in list_add_tail_lockless()
1066 * new->next == new. in list_add_tail_lockless()
1068 if (!try_cmpxchg(&new->next, &new, head)) in list_add_tail_lockless()
1072 * Initially ->next of a new element must be updated with the head in list_add_tail_lockless()
1074 * exchanged. XCHG guarantees memory ordering, thus ->next should be in list_add_tail_lockless()
1076 * swapped before prev->next is updated. in list_add_tail_lockless()
1079 prev = xchg(&head->prev, new); in list_add_tail_lockless()
1082 * It is safe to modify prev->next and new->prev, because a new element in list_add_tail_lockless()
1083 * is added only to the tail and new->next is updated before XCHG. in list_add_tail_lockless()
1086 prev->next = new; in list_add_tail_lockless()
1087 new->prev = prev; in list_add_tail_lockless()
1093 * Chains a new epi entry to the tail of the ep->ovflist in a lockless way,
1100 struct eventpoll *ep = epi->ep; in chain_epi_lockless()
1103 if (epi->next != EP_UNACTIVE_PTR) in chain_epi_lockless()
1107 if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR) in chain_epi_lockless()
1111 epi->next = xchg(&ep->ovflist, epi); in chain_epi_lockless()
1122 * events from another file descriptor, thus all modifications to ->rdllist
1123 * or ->ovflist are lockless. Read lock is paired with the write lock from
1128 * concurrently for the same @epi from different CPUs if poll table was inited
1138 struct eventpoll *ep = epi->ep; in ep_poll_callback()
1143 read_lock_irqsave(&ep->lock, flags); in ep_poll_callback()
1148 * If the event mask does not contain any poll(2) event, we consider the in ep_poll_callback()
1153 if (!(epi->event.events & ~EP_PRIVATE_BITS)) in ep_poll_callback()
1162 if (pollflags && !(pollflags & epi->event.events)) in ep_poll_callback()
1167 * (because we're accessing user memory, and because of linux f_op->poll() in ep_poll_callback()
1169 * chained in ep->ovflist and requeued later on. in ep_poll_callback()
1171 if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) { in ep_poll_callback()
1176 if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist)) in ep_poll_callback()
1181 * Wake up ( if active ) both the eventpoll wait list and the ->poll() in ep_poll_callback()
1184 if (waitqueue_active(&ep->wq)) { in ep_poll_callback()
1185 if ((epi->event.events & EPOLLEXCLUSIVE) && in ep_poll_callback()
1189 if (epi->event.events & EPOLLIN) in ep_poll_callback()
1193 if (epi->event.events & EPOLLOUT) in ep_poll_callback()
1201 wake_up(&ep->wq); in ep_poll_callback()
1203 if (waitqueue_active(&ep->poll_wait)) in ep_poll_callback()
1207 read_unlock_irqrestore(&ep->lock, flags); in ep_poll_callback()
1213 if (!(epi->event.events & EPOLLEXCLUSIVE)) in ep_poll_callback()
1219 * ->whead = NULL and do another remove_wait_queue() after in ep_poll_callback()
1222 list_del_init(&wait->entry); in ep_poll_callback()
1224 * ->whead != NULL protects us from the race with ep_free() in ep_poll_callback()
1225 * or ep_remove(), ep_remove_wait_queue() takes whead->lock in ep_poll_callback()
1229 smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL); in ep_poll_callback()
1243 struct epitem *epi = epq->epi; in ep_ptable_queue_proc()
1251 epq->epi = NULL; in ep_ptable_queue_proc()
1255 init_waitqueue_func_entry(&pwq->wait, ep_poll_callback); in ep_ptable_queue_proc()
1256 pwq->whead = whead; in ep_ptable_queue_proc()
1257 pwq->base = epi; in ep_ptable_queue_proc()
1258 if (epi->event.events & EPOLLEXCLUSIVE) in ep_ptable_queue_proc()
1259 add_wait_queue_exclusive(whead, &pwq->wait); in ep_ptable_queue_proc()
1261 add_wait_queue(whead, &pwq->wait); in ep_ptable_queue_proc()
1262 pwq->next = epi->pwqlist; in ep_ptable_queue_proc()
1263 epi->pwqlist = pwq; in ep_ptable_queue_proc()
1269 struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL; in ep_rbtree_insert()
1276 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); in ep_rbtree_insert()
1278 p = &parent->rb_right; in ep_rbtree_insert()
1281 p = &parent->rb_left; in ep_rbtree_insert()
1283 rb_link_node(&epi->rbn, parent, p); in ep_rbtree_insert()
1284 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost); in ep_rbtree_insert()
1311 return -1; in path_count_inc()
1329 return -1; in reverse_path_check_proc()
1333 struct hlist_head *refs = &epi->ep->refs; in reverse_path_check_proc()
1345 * reverse_path_check - The tfile_check_list is list of epitem_head, which have
1352 * %-1 otherwise.
1358 for (p = tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) { in reverse_path_check()
1362 error = reverse_path_check_proc(&p->epitems, 0); in reverse_path_check()
1375 if (!epi->ep->ws) { in ep_create_wakeup_source()
1376 epi->ep->ws = wakeup_source_register(NULL, "eventpoll"); in ep_create_wakeup_source()
1377 if (!epi->ep->ws) in ep_create_wakeup_source()
1378 return -ENOMEM; in ep_create_wakeup_source()
1381 take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry); in ep_create_wakeup_source()
1386 return -ENOMEM; in ep_create_wakeup_source()
1387 rcu_assign_pointer(epi->ws, ws); in ep_create_wakeup_source()
1397 RCU_INIT_POINTER(epi->ws, NULL); in ep_destroy_wakeup_source()
1415 ep = file->private_data; in attach_epitem()
1418 head = &ep->refs; in attach_epitem()
1419 } else if (!READ_ONCE(file->f_ep)) { in attach_epitem()
1423 return -ENOMEM; in attach_epitem()
1424 head = &to_free->epitems; in attach_epitem()
1426 spin_lock(&file->f_lock); in attach_epitem()
1427 if (!file->f_ep) { in attach_epitem()
1429 spin_unlock(&file->f_lock); in attach_epitem()
1432 file->f_ep = head; in attach_epitem()
1435 hlist_add_head_rcu(&epi->fllink, file->f_ep); in attach_epitem()
1436 spin_unlock(&file->f_lock); in attach_epitem()
1454 tep = tfile->private_data; in ep_insert()
1458 if (unlikely(percpu_counter_compare(&ep->user->epoll_watches, in ep_insert()
1460 return -ENOSPC; in ep_insert()
1461 percpu_counter_inc(&ep->user->epoll_watches); in ep_insert()
1464 percpu_counter_dec(&ep->user->epoll_watches); in ep_insert()
1465 return -ENOMEM; in ep_insert()
1469 INIT_LIST_HEAD(&epi->rdllink); in ep_insert()
1470 epi->ep = ep; in ep_insert()
1471 ep_set_ffd(&epi->ffd, tfile, fd); in ep_insert()
1472 epi->event = *event; in ep_insert()
1473 epi->next = EP_UNACTIVE_PTR; in ep_insert()
1476 mutex_lock_nested(&tep->mtx, 1); in ep_insert()
1480 mutex_unlock(&tep->mtx); in ep_insert()
1482 percpu_counter_dec(&ep->user->epoll_watches); in ep_insert()
1483 return -ENOMEM; in ep_insert()
1495 mutex_unlock(&tep->mtx); in ep_insert()
1500 return -EINVAL; in ep_insert()
1503 if (epi->event.events & EPOLLWAKEUP) { in ep_insert()
1511 /* Initialize the poll table using the queue callback */ in ep_insert()
1516 * Attach the item to the poll hooks and get current event bits. in ep_insert()
1519 * this operation completes, the poll callback can start hitting in ep_insert()
1525 * We have to check if something went wrong during the poll wait queue in ep_insert()
1531 return -ENOMEM; in ep_insert()
1535 write_lock_irq(&ep->lock); in ep_insert()
1542 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1546 if (waitqueue_active(&ep->wq)) in ep_insert()
1547 wake_up(&ep->wq); in ep_insert()
1548 if (waitqueue_active(&ep->poll_wait)) in ep_insert()
1552 write_unlock_irq(&ep->lock); in ep_insert()
1576 * Set the new event interest mask before calling f_op->poll(); in ep_modify()
1578 * f_op->poll() call and the new event set registering. in ep_modify()
1580 epi->event.events = event->events; /* need barrier below */ in ep_modify()
1581 epi->event.data = event->data; /* protected by mtx */ in ep_modify()
1582 if (epi->event.events & EPOLLWAKEUP) { in ep_modify()
1594 * event occurs immediately after we call f_op->poll(). in ep_modify()
1595 * We need this because we did not take ep->lock while in ep_modify()
1597 * ep->lock). in ep_modify()
1600 * when calling f_op->poll(). This barrier also in ep_modify()
1604 * This barrier will now guarantee ep_poll_callback or f_op->poll in ep_modify()
1616 write_lock_irq(&ep->lock); in ep_modify()
1618 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1622 if (waitqueue_active(&ep->wq)) in ep_modify()
1623 wake_up(&ep->wq); in ep_modify()
1624 if (waitqueue_active(&ep->poll_wait)) in ep_modify()
1627 write_unlock_irq(&ep->lock); in ep_modify()
1646 * Always short-circuit for fatal signals to allow threads to make a in ep_send_events()
1651 return -EINTR; in ep_send_events()
1655 mutex_lock(&ep->mtx); in ep_send_events()
1660 * Items cannot vanish during the loop we are holding ep->mtx. in ep_send_events()
1670 * Activate ep->ws before deactivating epi->ws to prevent in ep_send_events()
1671 * triggering auto-suspend here (in case we reactive epi->ws in ep_send_events()
1674 * This could be rearranged to delay the deactivation of epi->ws in ep_send_events()
1675 * instead, but then epi->ws would temporarily be out of sync in ep_send_events()
1680 if (ws->active) in ep_send_events()
1681 __pm_stay_awake(ep->ws); in ep_send_events()
1685 list_del_init(&epi->rdllink); in ep_send_events()
1688 * If the event mask intersect the caller-requested one, in ep_send_events()
1689 * deliver the event to userspace. Again, we are holding ep->mtx, in ep_send_events()
1696 events = epoll_put_uevent(revents, epi->event.data, events); in ep_send_events()
1698 list_add(&epi->rdllink, &txlist); in ep_send_events()
1701 res = -EFAULT; in ep_send_events()
1705 if (epi->event.events & EPOLLONESHOT) in ep_send_events()
1706 epi->event.events &= EP_PRIVATE_BITS; in ep_send_events()
1707 else if (!(epi->event.events & EPOLLET)) { in ep_send_events()
1714 * into ep->rdllist besides us. The epoll_ctl() in ep_send_events()
1717 * poll callback will queue them in ep->ovflist. in ep_send_events()
1719 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events()
1724 mutex_unlock(&ep->mtx); in ep_send_events()
1729 static struct timespec64 *ep_timeout_to_timespec(struct timespec64 *to, long ms) in ep_timeout_to_timespec() argument
1733 if (ms < 0) in ep_timeout_to_timespec()
1736 if (!ms) { in ep_timeout_to_timespec()
1737 to->tv_sec = 0; in ep_timeout_to_timespec()
1738 to->tv_nsec = 0; in ep_timeout_to_timespec()
1742 to->tv_sec = ms / MSEC_PER_SEC; in ep_timeout_to_timespec()
1743 to->tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC); in ep_timeout_to_timespec()
1761 list_del_init(&wq_entry->entry); in ep_autoremove_wake_function()
1766 * ep_poll - Retrieves ready events, and delivers them to the caller-supplied
1773 * @timeout: Maximum timeout for the ready events fetch operation, in
1774 * timespec. If the timeout is zero, the function will not block,
1775 * while if the @timeout ptr is NULL, the function will block
1783 int maxevents, struct timespec64 *timeout) in ep_poll() argument
1792 if (timeout && (timeout->tv_sec | timeout->tv_nsec)) { in ep_poll()
1793 slack = select_estimate_accuracy(timeout); in ep_poll()
1795 *to = timespec64_to_ktime(*timeout); in ep_poll()
1796 } else if (timeout) { in ep_poll()
1807 * with a non-zero timeout, this thread will check the ready list under in ep_poll()
1809 * timeout, the user by definition should not care and will have to in ep_poll()
1818 * 0 events and there's still timeout left over, we go in ep_poll()
1834 return -EINTR; in ep_poll()
1842 * lost. This is also good performance-wise, because on in ep_poll()
1844 * explicitly, thus ep->lock is not taken, which halts the in ep_poll()
1856 write_lock_irq(&ep->lock); in ep_poll()
1866 * plays with two lists (->rdllist and ->ovflist) and there in ep_poll()
1873 __add_wait_queue_exclusive(&ep->wq, &wait); in ep_poll()
1875 write_unlock_irq(&ep->lock); in ep_poll()
1890 write_lock_irq(&ep->lock); in ep_poll()
1894 * timeout expired before it could reacquire the lock. in ep_poll()
1900 __remove_wait_queue(&ep->wq, &wait); in ep_poll()
1901 write_unlock_irq(&ep->lock); in ep_poll()
1907 * ep_loop_check_proc - verify that adding an epoll file inside another
1916 * structure @ep does not violate the constraints, or %-1 otherwise.
1924 mutex_lock_nested(&ep->mtx, depth + 1); in ep_loop_check_proc()
1925 ep->gen = loop_check_gen; in ep_loop_check_proc()
1926 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { in ep_loop_check_proc()
1928 if (unlikely(is_file_epoll(epi->ffd.file))) { in ep_loop_check_proc()
1930 ep_tovisit = epi->ffd.file->private_data; in ep_loop_check_proc()
1931 if (ep_tovisit->gen == loop_check_gen) in ep_loop_check_proc()
1934 error = -1; in ep_loop_check_proc()
1948 list_file(epi->ffd.file); in ep_loop_check_proc()
1951 mutex_unlock(&ep->mtx); in ep_loop_check_proc()
1957 * ep_loop_check - Performs a check to verify that adding an epoll file (@to)
1965 * does not violate the constraints, or %-1 otherwise.
1978 tfile_check_list = head->next; in clear_tfile_check_list()
1997 return -EINVAL; in do_epoll_create()
2019 ep->file = file; in do_epoll_create()
2038 return -EINVAL; in SYSCALL_DEFINE1()
2052 return -EAGAIN; in epoll_mutex_lock()
2065 error = -EBADF; in do_epoll_ctl()
2075 /* The target file descriptor must support poll */ in do_epoll_ctl()
2076 error = -EPERM; in do_epoll_ctl()
2089 error = -EINVAL; in do_epoll_ctl()
2098 if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) { in do_epoll_ctl()
2102 (epds->events & ~EPOLLEXCLUSIVE_OK_BITS))) in do_epoll_ctl()
2110 ep = f.file->private_data; in do_epoll_ctl()
2127 error = epoll_mutex_lock(&ep->mtx, 0, nonblock); in do_epoll_ctl()
2131 if (READ_ONCE(f.file->f_ep) || ep->gen == loop_check_gen || in do_epoll_ctl()
2133 mutex_unlock(&ep->mtx); in do_epoll_ctl()
2140 tep = tf.file->private_data; in do_epoll_ctl()
2141 error = -ELOOP; in do_epoll_ctl()
2145 error = epoll_mutex_lock(&ep->mtx, 0, nonblock); in do_epoll_ctl()
2158 error = -EINVAL; in do_epoll_ctl()
2162 epds->events |= EPOLLERR | EPOLLHUP; in do_epoll_ctl()
2165 error = -EEXIST; in do_epoll_ctl()
2171 error = -ENOENT; in do_epoll_ctl()
2175 if (!(epi->event.events & EPOLLEXCLUSIVE)) { in do_epoll_ctl()
2176 epds->events |= EPOLLERR | EPOLLHUP; in do_epoll_ctl()
2180 error = -ENOENT; in do_epoll_ctl()
2183 mutex_unlock(&ep->mtx); in do_epoll_ctl()
2212 return -EFAULT; in SYSCALL_DEFINE4()
2230 return -EINVAL; in do_epoll_wait()
2234 return -EFAULT; in do_epoll_wait()
2239 return -EBADF; in do_epoll_wait()
2245 error = -EINVAL; in do_epoll_wait()
2253 ep = f.file->private_data; in do_epoll_wait()
2264 int, maxevents, int, timeout) in SYSCALL_DEFINE4() argument
2269 ep_timeout_to_timespec(&to, timeout)); in SYSCALL_DEFINE4()
2292 restore_saved_sigmask_unless(error == -EINTR); in do_epoll_pwait()
2298 int, maxevents, int, timeout, const sigset_t __user *, sigmask, in SYSCALL_DEFINE6() argument
2304 ep_timeout_to_timespec(&to, timeout), in SYSCALL_DEFINE6()
2309 int, maxevents, const struct __kernel_timespec __user *, timeout, in SYSCALL_DEFINE6() argument
2314 if (timeout) { in SYSCALL_DEFINE6()
2315 if (get_timespec64(&ts, timeout)) in SYSCALL_DEFINE6()
2316 return -EFAULT; in SYSCALL_DEFINE6()
2319 return -EINVAL; in SYSCALL_DEFINE6()
2328 int maxevents, struct timespec64 *timeout, in do_compat_epoll_pwait() argument
2342 err = do_epoll_wait(epfd, events, maxevents, timeout); in do_compat_epoll_pwait()
2344 restore_saved_sigmask_unless(err == -EINTR); in do_compat_epoll_pwait()
2351 int, maxevents, int, timeout, in COMPAT_SYSCALL_DEFINE6() argument
2358 ep_timeout_to_timespec(&to, timeout), in COMPAT_SYSCALL_DEFINE6()
2365 const struct __kernel_timespec __user *, timeout, in COMPAT_SYSCALL_DEFINE6() argument
2371 if (timeout) { in COMPAT_SYSCALL_DEFINE6()
2372 if (get_timespec64(&ts, timeout)) in COMPAT_SYSCALL_DEFINE6()
2373 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
2376 return -EINVAL; in COMPAT_SYSCALL_DEFINE6()
2393 max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) / in eventpoll_init()
2399 * using an extra cache line on 64-bit (and smaller) CPUs in eventpoll_init()