251 	struct epitem *epi;  member
335 static inline int ep_is_linked(struct epitem *epi)  in ep_is_linked()  argument
337 	return !list_empty(&epi->rdllink);  in ep_is_linked()
354 	return container_of(p, struct ep_pqueue, pt)->epi;  in ep_item_from_epqueue()
415 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)  in ep_set_busy_poll_napi_id()  argument
426 	sock = sock_from_file(epi->ffd.file, &err);  in ep_set_busy_poll_napi_id()
435 	ep = epi->ep;  in ep_set_busy_poll_napi_id()
458 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)  in ep_set_busy_poll_napi_id()  argument
609 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)  in ep_unregister_pollwait()  argument
611 	struct list_head *lsthead = &epi->pwqlist;  in ep_unregister_pollwait()
624 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)  in ep_wakeup_source()  argument
626 	return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));  in ep_wakeup_source()
630 static inline void ep_pm_stay_awake(struct epitem *epi)  in ep_pm_stay_awake()  argument
632 	struct wakeup_source *ws = ep_wakeup_source(epi);  in ep_pm_stay_awake()
638 static inline bool ep_has_wakeup_source(struct epitem *epi)  in ep_has_wakeup_source()  argument
640 	return rcu_access_pointer(epi->ws) ? true : false;  in ep_has_wakeup_source()
644 static inline void ep_pm_stay_awake_rcu(struct epitem *epi)  in ep_pm_stay_awake_rcu()  argument
649 	ws = rcu_dereference(epi->ws);  in ep_pm_stay_awake_rcu()
675 	struct epitem *epi, *nepi;  in ep_scan_ready_list()  local
712 	for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;  in ep_scan_ready_list()
713 	     nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {  in ep_scan_ready_list()
720 		if (!ep_is_linked(epi)) {  in ep_scan_ready_list()
725 			list_add(&epi->rdllink, &ep->rdllist);  in ep_scan_ready_list()
726 			ep_pm_stay_awake(epi);  in ep_scan_ready_list()
766 	struct epitem *epi = container_of(head, struct epitem, rcu);  in epi_rcu_free()  local
767 	kmem_cache_free(epi_cache, epi);  in epi_rcu_free()
774 static int ep_remove(struct eventpoll *ep, struct epitem *epi)  in ep_remove()  argument
776 	struct file *file = epi->ffd.file;  in ep_remove()
783 	ep_unregister_pollwait(ep, epi);  in ep_remove()
787 	list_del_rcu(&epi->fllink);  in ep_remove()
790 	rb_erase_cached(&epi->rbn, &ep->rbr);  in ep_remove()
793 	if (ep_is_linked(epi))  in ep_remove()
794 		list_del_init(&epi->rdllink);  in ep_remove()
797 	wakeup_source_unregister(ep_wakeup_source(epi));  in ep_remove()
805 	call_rcu(&epi->rcu, epi_rcu_free);  in ep_remove()
815 	struct epitem *epi;  in ep_free()  local
835 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
837 		ep_unregister_pollwait(ep, epi);  in ep_free()
851 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
852 		ep_remove(ep, epi);  in ep_free()
884 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,  in ep_item_poll()  argument
890 	pt->_key = epi->event.events;  in ep_item_poll()
891 	if (!is_file_epoll(epi->ffd.file))  in ep_item_poll()
892 		return vfs_poll(epi->ffd.file, pt) & epi->event.events;  in ep_item_poll()
894 	ep = epi->ffd.file->private_data;  in ep_item_poll()
895 	poll_wait(epi->ffd.file, &ep->poll_wait, pt);  in ep_item_poll()
898 	return ep_scan_ready_list(epi->ffd.file->private_data,  in ep_item_poll()
900 				  locked) & epi->event.events;  in ep_item_poll()
906 	struct epitem *epi, *tmp;  in ep_read_events_proc()  local
913 	list_for_each_entry_safe(epi, tmp, head, rdllink) {  in ep_read_events_proc()
914 		if (ep_item_poll(epi, &pt, depth)) {  in ep_read_events_proc()
922 			__pm_relax(ep_wakeup_source(epi));  in ep_read_events_proc()
923 			list_del_init(&epi->rdllink);  in ep_read_events_proc()
954 		struct epitem *epi = rb_entry(rbp, struct epitem, rbn);  in ep_show_fdinfo()  local
955 		struct inode *inode = file_inode(epi->ffd.file);  in ep_show_fdinfo()
959 			   epi->ffd.fd, epi->event.events,  in ep_show_fdinfo()
960 			   (long long)epi->event.data,  in ep_show_fdinfo()
961 			   (long long)epi->ffd.file->f_pos,  in ep_show_fdinfo()
988 	struct epitem *epi, *next;  in eventpoll_release_file()  local
1004 	list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) {  in eventpoll_release_file()
1005 		ep = epi->ep;  in eventpoll_release_file()
1007 		ep_remove(ep, epi);  in eventpoll_release_file()
1052 	struct epitem *epi, *epir = NULL;  in ep_find()  local
1057 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_find()
1058 		kcmp = ep_cmp_ffd(&ffd, &epi->ffd);  in ep_find()
1064 			epir = epi;  in ep_find()
1076 	struct epitem *epi;  in ep_find_tfd()  local
1079 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_find_tfd()
1080 		if (epi->ffd.fd == tfd) {  in ep_find_tfd()
1082 				return epi;  in ep_find_tfd()
1097 	struct epitem *epi;  in get_epoll_tfile_raw_ptr()  local
1105 	epi = ep_find_tfd(ep, tfd, toff);  in get_epoll_tfile_raw_ptr()
1106 	if (epi)  in get_epoll_tfile_raw_ptr()
1107 		file_raw = epi->ffd.file;  in get_epoll_tfile_raw_ptr()
1175 static inline bool chain_epi_lockless(struct epitem *epi)  in chain_epi_lockless()  argument
1177 	struct eventpoll *ep = epi->ep;  in chain_epi_lockless()
1180 	if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)  in chain_epi_lockless()
1184 	epi->next = xchg(&ep->ovflist, epi);  in chain_epi_lockless()
1210 	struct epitem *epi = ep_item_from_wait(wait);  in ep_poll_callback()  local
1211 	struct eventpoll *ep = epi->ep;  in ep_poll_callback()
1218 	ep_set_busy_poll_napi_id(epi);  in ep_poll_callback()
1226 	if (!(epi->event.events & ~EP_PRIVATE_BITS))  in ep_poll_callback()
1235 	if (pollflags && !(pollflags & epi->event.events))  in ep_poll_callback()
1245 		if (epi->next == EP_UNACTIVE_PTR &&  in ep_poll_callback()
1246 		    chain_epi_lockless(epi))  in ep_poll_callback()
1247 			ep_pm_stay_awake_rcu(epi);  in ep_poll_callback()
1252 	if (!ep_is_linked(epi) &&  in ep_poll_callback()
1253 	    list_add_tail_lockless(&epi->rdllink, &ep->rdllist)) {  in ep_poll_callback()
1254 		ep_pm_stay_awake_rcu(epi);  in ep_poll_callback()
1262 		if ((epi->event.events & EPOLLEXCLUSIVE) &&  in ep_poll_callback()
1266 				if (epi->event.events & EPOLLIN)  in ep_poll_callback()
1270 				if (epi->event.events & EPOLLOUT)  in ep_poll_callback()
1290 	if (!(epi->event.events & EPOLLEXCLUSIVE))  in ep_poll_callback()
1319 	struct epitem *epi = ep_item_from_epqueue(pt);  in ep_ptable_queue_proc()  local
1322 	if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) {  in ep_ptable_queue_proc()
1325 		pwq->base = epi;  in ep_ptable_queue_proc()
1326 		if (epi->event.events & EPOLLEXCLUSIVE)  in ep_ptable_queue_proc()
1330 		list_add_tail(&pwq->llink, &epi->pwqlist);  in ep_ptable_queue_proc()
1331 		epi->nwait++;  in ep_ptable_queue_proc()
1334 		epi->nwait = -1;  in ep_ptable_queue_proc()
1338 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)  in ep_rbtree_insert()  argument
1348 		kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);  in ep_rbtree_insert()
1355 	rb_link_node(&epi->rbn, parent, p);  in ep_rbtree_insert()
1356 	rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);  in ep_rbtree_insert()
1400 	struct epitem *epi;  in reverse_path_check_proc()  local
1404 	list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) {  in reverse_path_check_proc()
1405 		child_file = epi->ep->file;  in reverse_path_check_proc()
1456 static int ep_create_wakeup_source(struct epitem *epi)  in ep_create_wakeup_source()  argument
1461 	if (!epi->ep->ws) {  in ep_create_wakeup_source()
1462 		epi->ep->ws = wakeup_source_register(NULL, "eventpoll");  in ep_create_wakeup_source()
1463 		if (!epi->ep->ws)  in ep_create_wakeup_source()
1467 	name = epi->ffd.file->f_path.dentry->d_name.name;  in ep_create_wakeup_source()
1472 	rcu_assign_pointer(epi->ws, ws);  in ep_create_wakeup_source()
1478 static noinline void ep_destroy_wakeup_source(struct epitem *epi)  in ep_destroy_wakeup_source()  argument
1480 	struct wakeup_source *ws = ep_wakeup_source(epi);  in ep_destroy_wakeup_source()
1482 	RCU_INIT_POINTER(epi->ws, NULL);  in ep_destroy_wakeup_source()
1502 	struct epitem *epi;  in ep_insert()  local
1510 	if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))  in ep_insert()
1514 	INIT_LIST_HEAD(&epi->rdllink);  in ep_insert()
1515 	INIT_LIST_HEAD(&epi->fllink);  in ep_insert()
1516 	INIT_LIST_HEAD(&epi->pwqlist);  in ep_insert()
1517 	epi->ep = ep;  in ep_insert()
1518 	ep_set_ffd(&epi->ffd, tfile, fd);  in ep_insert()
1519 	epi->event = *event;  in ep_insert()
1520 	epi->nwait = 0;  in ep_insert()
1521 	epi->next = EP_UNACTIVE_PTR;  in ep_insert()
1522 	if (epi->event.events & EPOLLWAKEUP) {  in ep_insert()
1523 		error = ep_create_wakeup_source(epi);  in ep_insert()
1527 		RCU_INIT_POINTER(epi->ws, NULL);  in ep_insert()
1531 	epq.epi = epi;  in ep_insert()
1541 	revents = ep_item_poll(epi, &epq.pt, 1);  in ep_insert()
1549 	if (epi->nwait < 0)  in ep_insert()
1554 	list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);  in ep_insert()
1561 	ep_rbtree_insert(ep, epi);  in ep_insert()
1572 	ep_set_busy_poll_napi_id(epi);  in ep_insert()
1575 	if (revents && !ep_is_linked(epi)) {  in ep_insert()
1576 		list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_insert()
1577 		ep_pm_stay_awake(epi);  in ep_insert()
1598 	list_del_rcu(&epi->fllink);  in ep_insert()
1601 	rb_erase_cached(&epi->rbn, &ep->rbr);  in ep_insert()
1604 	ep_unregister_pollwait(ep, epi);  in ep_insert()
1613 	if (ep_is_linked(epi))  in ep_insert()
1614 		list_del_init(&epi->rdllink);  in ep_insert()
1617 	wakeup_source_unregister(ep_wakeup_source(epi));  in ep_insert()
1620 	kmem_cache_free(epi_cache, epi);  in ep_insert()
1629 static int ep_modify(struct eventpoll *ep, struct epitem *epi,  in ep_modify()  argument
1644 	epi->event.events = event->events; /* need barrier below */  in ep_modify()
1645 	epi->event.data = event->data; /* protected by mtx */  in ep_modify()
1646 	if (epi->event.events & EPOLLWAKEUP) {  in ep_modify()
1647 		if (!ep_has_wakeup_source(epi))  in ep_modify()
1648 			ep_create_wakeup_source(epi);  in ep_modify()
1649 	} else if (ep_has_wakeup_source(epi)) {  in ep_modify()
1650 		ep_destroy_wakeup_source(epi);  in ep_modify()
1679 	if (ep_item_poll(epi, &pt, 1)) {  in ep_modify()
1681 		if (!ep_is_linked(epi)) {  in ep_modify()
1682 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_modify()
1683 			ep_pm_stay_awake(epi);  in ep_modify()
1706 	struct epitem *epi, *tmp;  in ep_send_events_proc()  local
1721 	list_for_each_entry_safe(epi, tmp, head, rdllink) {  in ep_send_events_proc()
1734 		ws = ep_wakeup_source(epi);  in ep_send_events_proc()
1741 		list_del_init(&epi->rdllink);  in ep_send_events_proc()
1749 		revents = ep_item_poll(epi, &pt, 1);  in ep_send_events_proc()
1754 		    __put_user(epi->event.data, &uevent->data)) {  in ep_send_events_proc()
1755 			list_add(&epi->rdllink, head);  in ep_send_events_proc()
1756 			ep_pm_stay_awake(epi);  in ep_send_events_proc()
1763 		if (epi->event.events & EPOLLONESHOT)  in ep_send_events_proc()
1764 			epi->event.events &= EP_PRIVATE_BITS;  in ep_send_events_proc()
1765 		else if (!(epi->event.events & EPOLLET)) {  in ep_send_events_proc()
1777 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_send_events_proc()
1778 			ep_pm_stay_awake(epi);  in ep_send_events_proc()
1964 	struct epitem *epi;  in ep_loop_check_proc()  local
1970 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_loop_check_proc()
1971 		if (unlikely(is_file_epoll(epi->ffd.file))) {  in ep_loop_check_proc()
1972 			ep_tovisit = epi->ffd.file->private_data;  in ep_loop_check_proc()
1976 					ep_loop_check_proc, epi->ffd.file,  in ep_loop_check_proc()
1989 			if (list_empty(&epi->ffd.file->f_tfile_llink))  in ep_loop_check_proc()
1990 				list_add(&epi->ffd.file->f_tfile_llink,  in ep_loop_check_proc()
2110 	struct epitem *epi;  in SYSCALL_DEFINE4()  local
2210 	epi = ep_find(ep, tf.file, fd);  in SYSCALL_DEFINE4()
2215 		if (!epi) {  in SYSCALL_DEFINE4()
2224 		if (epi)  in SYSCALL_DEFINE4()
2225 			error = ep_remove(ep, epi);  in SYSCALL_DEFINE4()
2230 		if (epi) {  in SYSCALL_DEFINE4()
2231 			if (!(epi->event.events & EPOLLEXCLUSIVE)) {  in SYSCALL_DEFINE4()
2233 				error = ep_modify(ep, epi, &epds);  in SYSCALL_DEFINE4()