Lines Matching refs:epi
234 struct epitem *epi; member
293 struct epitem *epi= container_of(p, struct epitem, fllink); in unlist_file() local
294 spin_lock(&epi->ffd.file->f_lock); in unlist_file()
298 spin_unlock(&epi->ffd.file->f_lock); in unlist_file()
355 static inline int ep_is_linked(struct epitem *epi) in ep_is_linked() argument
357 return !list_empty(&epi->rdllink); in ep_is_linked()
422 static inline void ep_set_busy_poll_napi_id(struct epitem *epi) in ep_set_busy_poll_napi_id() argument
432 sock = sock_from_file(epi->ffd.file); in ep_set_busy_poll_napi_id()
441 ep = epi->ep; in ep_set_busy_poll_napi_id()
461 static inline void ep_set_busy_poll_napi_id(struct epitem *epi) in ep_set_busy_poll_napi_id() argument
494 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) in ep_poll_safewake() argument
515 if (epi) { in ep_poll_safewake()
516 if ((is_file_epoll(epi->ffd.file))) { in ep_poll_safewake()
517 ep_src = epi->ffd.file->private_data; in ep_poll_safewake()
532 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) in ep_poll_safewake() argument
561 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) in ep_unregister_pollwait() argument
563 struct eppoll_entry **p = &epi->pwqlist; in ep_unregister_pollwait()
574 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi) in ep_wakeup_source() argument
576 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); in ep_wakeup_source()
580 static inline void ep_pm_stay_awake(struct epitem *epi) in ep_pm_stay_awake() argument
582 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_pm_stay_awake()
588 static inline bool ep_has_wakeup_source(struct epitem *epi) in ep_has_wakeup_source() argument
590 return rcu_access_pointer(epi->ws) ? true : false; in ep_has_wakeup_source()
594 static inline void ep_pm_stay_awake_rcu(struct epitem *epi) in ep_pm_stay_awake_rcu() argument
599 ws = rcu_dereference(epi->ws); in ep_pm_stay_awake_rcu()
630 struct epitem *epi, *nepi; in ep_done_scan() local
638 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; in ep_done_scan()
639 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) { in ep_done_scan()
646 if (!ep_is_linked(epi)) { in ep_done_scan()
651 list_add(&epi->rdllink, &ep->rdllist); in ep_done_scan()
652 ep_pm_stay_awake(epi); in ep_done_scan()
678 struct epitem *epi = container_of(head, struct epitem, rcu); in epi_rcu_free() local
679 kmem_cache_free(epi_cache, epi); in epi_rcu_free()
686 static int ep_remove(struct eventpoll *ep, struct epitem *epi) in ep_remove() argument
688 struct file *file = epi->ffd.file; in ep_remove()
697 ep_unregister_pollwait(ep, epi); in ep_remove()
703 if (head->first == &epi->fllink && !epi->fllink.next) { in ep_remove()
712 hlist_del_rcu(&epi->fllink); in ep_remove()
716 rb_erase_cached(&epi->rbn, &ep->rbr); in ep_remove()
719 if (ep_is_linked(epi)) in ep_remove()
720 list_del_init(&epi->rdllink); in ep_remove()
723 wakeup_source_unregister(ep_wakeup_source(epi)); in ep_remove()
731 call_rcu(&epi->rcu, epi_rcu_free); in ep_remove()
741 struct epitem *epi; in ep_free() local
761 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
763 ep_unregister_pollwait(ep, epi); in ep_free()
777 epi = rb_entry(rbp, struct epitem, rbn); in ep_free()
778 ep_remove(ep, epi); in ep_free()
800 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth);
806 struct epitem *epi, *tmp; in __ep_eventpoll_poll() local
821 list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { in __ep_eventpoll_poll()
822 if (ep_item_poll(epi, &pt, depth + 1)) { in __ep_eventpoll_poll()
831 __pm_relax(ep_wakeup_source(epi)); in __ep_eventpoll_poll()
832 list_del_init(&epi->rdllink); in __ep_eventpoll_poll()
845 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, in ep_item_poll() argument
848 struct file *file = epi->ffd.file; in ep_item_poll()
851 pt->_key = epi->event.events; in ep_item_poll()
856 return res & epi->event.events; in ep_item_poll()
872 struct epitem *epi = rb_entry(rbp, struct epitem, rbn); in ep_show_fdinfo() local
873 struct inode *inode = file_inode(epi->ffd.file); in ep_show_fdinfo()
877 epi->ffd.fd, epi->event.events, in ep_show_fdinfo()
878 (long long)epi->event.data, in ep_show_fdinfo()
879 (long long)epi->ffd.file->f_pos, in ep_show_fdinfo()
906 struct epitem *epi; in eventpoll_release_file() local
927 hlist_for_each_entry_safe(epi, next, file->f_ep, fllink) { in eventpoll_release_file()
928 ep = epi->ep; in eventpoll_release_file()
930 ep_remove(ep, epi); in eventpoll_release_file()
975 struct epitem *epi, *epir = NULL; in ep_find() local
980 epi = rb_entry(rbp, struct epitem, rbn); in ep_find()
981 kcmp = ep_cmp_ffd(&ffd, &epi->ffd); in ep_find()
987 epir = epi; in ep_find()
999 struct epitem *epi; in ep_find_tfd() local
1002 epi = rb_entry(rbp, struct epitem, rbn); in ep_find_tfd()
1003 if (epi->ffd.fd == tfd) { in ep_find_tfd()
1005 return epi; in ep_find_tfd()
1020 struct epitem *epi; in get_epoll_tfile_raw_ptr() local
1028 epi = ep_find_tfd(ep, tfd, toff); in get_epoll_tfile_raw_ptr()
1029 if (epi) in get_epoll_tfile_raw_ptr()
1030 file_raw = epi->ffd.file; in get_epoll_tfile_raw_ptr()
1098 static inline bool chain_epi_lockless(struct epitem *epi) in chain_epi_lockless() argument
1100 struct eventpoll *ep = epi->ep; in chain_epi_lockless()
1103 if (epi->next != EP_UNACTIVE_PTR) in chain_epi_lockless()
1107 if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR) in chain_epi_lockless()
1111 epi->next = xchg(&ep->ovflist, epi); in chain_epi_lockless()
1137 struct epitem *epi = ep_item_from_wait(wait); in ep_poll_callback() local
1138 struct eventpoll *ep = epi->ep; in ep_poll_callback()
1145 ep_set_busy_poll_napi_id(epi); in ep_poll_callback()
1153 if (!(epi->event.events & ~EP_PRIVATE_BITS)) in ep_poll_callback()
1162 if (pollflags && !(pollflags & epi->event.events)) in ep_poll_callback()
1172 if (chain_epi_lockless(epi)) in ep_poll_callback()
1173 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1174 } else if (!ep_is_linked(epi)) { in ep_poll_callback()
1176 if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist)) in ep_poll_callback()
1177 ep_pm_stay_awake_rcu(epi); in ep_poll_callback()
1185 if ((epi->event.events & EPOLLEXCLUSIVE) && in ep_poll_callback()
1189 if (epi->event.events & EPOLLIN) in ep_poll_callback()
1193 if (epi->event.events & EPOLLOUT) in ep_poll_callback()
1211 ep_poll_safewake(ep, epi); in ep_poll_callback()
1213 if (!(epi->event.events & EPOLLEXCLUSIVE)) in ep_poll_callback()
1243 struct epitem *epi = epq->epi; in ep_ptable_queue_proc() local
1246 if (unlikely(!epi)) // an earlier allocation has failed in ep_ptable_queue_proc()
1251 epq->epi = NULL; in ep_ptable_queue_proc()
1257 pwq->base = epi; in ep_ptable_queue_proc()
1258 if (epi->event.events & EPOLLEXCLUSIVE) in ep_ptable_queue_proc()
1262 pwq->next = epi->pwqlist; in ep_ptable_queue_proc()
1263 epi->pwqlist = pwq; in ep_ptable_queue_proc()
1266 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) in ep_rbtree_insert() argument
1276 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); in ep_rbtree_insert()
1283 rb_link_node(&epi->rbn, parent, p); in ep_rbtree_insert()
1284 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost); in ep_rbtree_insert()
1326 struct epitem *epi; in reverse_path_check_proc() local
1332 hlist_for_each_entry_rcu(epi, refs, fllink) { in reverse_path_check_proc()
1333 struct hlist_head *refs = &epi->ep->refs; in reverse_path_check_proc()
1370 static int ep_create_wakeup_source(struct epitem *epi) in ep_create_wakeup_source() argument
1375 if (!epi->ep->ws) { in ep_create_wakeup_source()
1376 epi->ep->ws = wakeup_source_register(NULL, "eventpoll"); in ep_create_wakeup_source()
1377 if (!epi->ep->ws) in ep_create_wakeup_source()
1381 take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry); in ep_create_wakeup_source()
1387 rcu_assign_pointer(epi->ws, ws); in ep_create_wakeup_source()
1393 static noinline void ep_destroy_wakeup_source(struct epitem *epi) in ep_destroy_wakeup_source() argument
1395 struct wakeup_source *ws = ep_wakeup_source(epi); in ep_destroy_wakeup_source()
1397 RCU_INIT_POINTER(epi->ws, NULL); in ep_destroy_wakeup_source()
1408 static int attach_epitem(struct file *file, struct epitem *epi) in attach_epitem() argument
1435 hlist_add_head_rcu(&epi->fllink, file->f_ep); in attach_epitem()
1449 struct epitem *epi; in ep_insert() local
1463 if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) { in ep_insert()
1469 INIT_LIST_HEAD(&epi->rdllink); in ep_insert()
1470 epi->ep = ep; in ep_insert()
1471 ep_set_ffd(&epi->ffd, tfile, fd); in ep_insert()
1472 epi->event = *event; in ep_insert()
1473 epi->next = EP_UNACTIVE_PTR; in ep_insert()
1478 if (unlikely(attach_epitem(tfile, epi) < 0)) { in ep_insert()
1481 kmem_cache_free(epi_cache, epi); in ep_insert()
1493 ep_rbtree_insert(ep, epi); in ep_insert()
1499 ep_remove(ep, epi); in ep_insert()
1503 if (epi->event.events & EPOLLWAKEUP) { in ep_insert()
1504 error = ep_create_wakeup_source(epi); in ep_insert()
1506 ep_remove(ep, epi); in ep_insert()
1512 epq.epi = epi; in ep_insert()
1522 revents = ep_item_poll(epi, &epq.pt, 1); in ep_insert()
1529 if (unlikely(!epq.epi)) { in ep_insert()
1530 ep_remove(ep, epi); in ep_insert()
1538 ep_set_busy_poll_napi_id(epi); in ep_insert()
1541 if (revents && !ep_is_linked(epi)) { in ep_insert()
1542 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_insert()
1543 ep_pm_stay_awake(epi); in ep_insert()
1565 static int ep_modify(struct eventpoll *ep, struct epitem *epi, in ep_modify() argument
1580 epi->event.events = event->events; /* need barrier below */ in ep_modify()
1581 epi->event.data = event->data; /* protected by mtx */ in ep_modify()
1582 if (epi->event.events & EPOLLWAKEUP) { in ep_modify()
1583 if (!ep_has_wakeup_source(epi)) in ep_modify()
1584 ep_create_wakeup_source(epi); in ep_modify()
1585 } else if (ep_has_wakeup_source(epi)) { in ep_modify()
1586 ep_destroy_wakeup_source(epi); in ep_modify()
1615 if (ep_item_poll(epi, &pt, 1)) { in ep_modify()
1617 if (!ep_is_linked(epi)) { in ep_modify()
1618 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_modify()
1619 ep_pm_stay_awake(epi); in ep_modify()
1640 struct epitem *epi, *tmp; in ep_send_events() local
1662 list_for_each_entry_safe(epi, tmp, &txlist, rdllink) { in ep_send_events()
1678 ws = ep_wakeup_source(epi); in ep_send_events()
1685 list_del_init(&epi->rdllink); in ep_send_events()
1692 revents = ep_item_poll(epi, &pt, 1); in ep_send_events()
1696 events = epoll_put_uevent(revents, epi->event.data, events); in ep_send_events()
1698 list_add(&epi->rdllink, &txlist); in ep_send_events()
1699 ep_pm_stay_awake(epi); in ep_send_events()
1705 if (epi->event.events & EPOLLONESHOT) in ep_send_events()
1706 epi->event.events &= EP_PRIVATE_BITS; in ep_send_events()
1707 else if (!(epi->event.events & EPOLLET)) { in ep_send_events()
1719 list_add_tail(&epi->rdllink, &ep->rdllist); in ep_send_events()
1720 ep_pm_stay_awake(epi); in ep_send_events()
1922 struct epitem *epi; in ep_loop_check_proc() local
1927 epi = rb_entry(rbp, struct epitem, rbn); in ep_loop_check_proc()
1928 if (unlikely(is_file_epoll(epi->ffd.file))) { in ep_loop_check_proc()
1930 ep_tovisit = epi->ffd.file->private_data; in ep_loop_check_proc()
1948 list_file(epi->ffd.file); in ep_loop_check_proc()
2062 struct epitem *epi; in do_epoll_ctl() local
2156 epi = ep_find(ep, tf.file, fd); in do_epoll_ctl()
2161 if (!epi) { in do_epoll_ctl()
2168 if (epi) in do_epoll_ctl()
2169 error = ep_remove(ep, epi); in do_epoll_ctl()
2174 if (epi) { in do_epoll_ctl()
2175 if (!(epi->event.events & EPOLLEXCLUSIVE)) { in do_epoll_ctl()
2177 error = ep_modify(ep, epi, epds); in do_epoll_ctl()