255 	struct epitem *epi;  member
338 static inline int ep_is_linked(struct epitem *epi)  in ep_is_linked()  argument
340 	return !list_empty(&epi->rdllink);  in ep_is_linked()
357 	return container_of(p, struct ep_pqueue, pt)->epi;  in ep_item_from_epqueue()
412 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)  in ep_set_busy_poll_napi_id()  argument
423 	sock = sock_from_file(epi->ffd.file, &err);  in ep_set_busy_poll_napi_id()
432 	ep = epi->ep;  in ep_set_busy_poll_napi_id()
455 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)  in ep_set_busy_poll_napi_id()  argument
551 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi)  in ep_poll_safewake()  argument
572 	if (epi) {  in ep_poll_safewake()
573 		if ((is_file_epoll(epi->ffd.file))) {  in ep_poll_safewake()
574 			ep_src = epi->ffd.file->private_data;  in ep_poll_safewake()
589 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi)  in ep_poll_safewake()  argument
618 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)  in ep_unregister_pollwait()  argument
620 	struct list_head *lsthead = &epi->pwqlist;  in ep_unregister_pollwait()
633 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)  in ep_wakeup_source()  argument
635 	return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));  in ep_wakeup_source()
639 static inline void ep_pm_stay_awake(struct epitem *epi)  in ep_pm_stay_awake()  argument
641 	struct wakeup_source *ws = ep_wakeup_source(epi);  in ep_pm_stay_awake()
647 static inline bool ep_has_wakeup_source(struct epitem *epi)  in ep_has_wakeup_source()  argument
649 	return rcu_access_pointer(epi->ws) ? true : false;  in ep_has_wakeup_source()
653 static inline void ep_pm_stay_awake_rcu(struct epitem *epi)  in ep_pm_stay_awake_rcu()  argument
658 	ws = rcu_dereference(epi->ws);  in ep_pm_stay_awake_rcu()
683 	struct epitem *epi, *nepi;  in ep_scan_ready_list()  local
720 	for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;  in ep_scan_ready_list()
721 	     nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {  in ep_scan_ready_list()
728 		if (!ep_is_linked(epi)) {  in ep_scan_ready_list()
733 			list_add(&epi->rdllink, &ep->rdllist);  in ep_scan_ready_list()
734 			ep_pm_stay_awake(epi);  in ep_scan_ready_list()
759 	struct epitem *epi = container_of(head, struct epitem, rcu);  in epi_rcu_free()  local
760 	kmem_cache_free(epi_cache, epi);  in epi_rcu_free()
767 static int ep_remove(struct eventpoll *ep, struct epitem *epi)  in ep_remove()  argument
769 	struct file *file = epi->ffd.file;  in ep_remove()
776 	ep_unregister_pollwait(ep, epi);  in ep_remove()
780 	list_del_rcu(&epi->fllink);  in ep_remove()
783 	rb_erase_cached(&epi->rbn, &ep->rbr);  in ep_remove()
786 	if (ep_is_linked(epi))  in ep_remove()
787 		list_del_init(&epi->rdllink);  in ep_remove()
790 	wakeup_source_unregister(ep_wakeup_source(epi));  in ep_remove()
798 	call_rcu(&epi->rcu, epi_rcu_free);  in ep_remove()
808 	struct epitem *epi;  in ep_free()  local
828 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
830 		ep_unregister_pollwait(ep, epi);  in ep_free()
844 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
845 		ep_remove(ep, epi);  in ep_free()
877 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,  in ep_item_poll()  argument
883 	pt->_key = epi->event.events;  in ep_item_poll()
884 	if (!is_file_epoll(epi->ffd.file))  in ep_item_poll()
885 		return vfs_poll(epi->ffd.file, pt) & epi->event.events;  in ep_item_poll()
887 	ep = epi->ffd.file->private_data;  in ep_item_poll()
888 	poll_wait(epi->ffd.file, &ep->poll_wait, pt);  in ep_item_poll()
891 	return ep_scan_ready_list(epi->ffd.file->private_data,  in ep_item_poll()
893 				  locked) & epi->event.events;  in ep_item_poll()
899 	struct epitem *epi, *tmp;  in ep_read_events_proc()  local
906 	list_for_each_entry_safe(epi, tmp, head, rdllink) {  in ep_read_events_proc()
907 		if (ep_item_poll(epi, &pt, depth)) {  in ep_read_events_proc()
915 			__pm_relax(ep_wakeup_source(epi));  in ep_read_events_proc()
916 			list_del_init(&epi->rdllink);  in ep_read_events_proc()
947 		struct epitem *epi = rb_entry(rbp, struct epitem, rbn);  in ep_show_fdinfo()  local
948 		struct inode *inode = file_inode(epi->ffd.file);  in ep_show_fdinfo()
952 			   epi->ffd.fd, epi->event.events,  in ep_show_fdinfo()
953 			   (long long)epi->event.data,  in ep_show_fdinfo()
954 			   (long long)epi->ffd.file->f_pos,  in ep_show_fdinfo()
981 	struct epitem *epi, *next;  in eventpoll_release_file()  local
997 	list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) {  in eventpoll_release_file()
998 		ep = epi->ep;  in eventpoll_release_file()
1000 		ep_remove(ep, epi);  in eventpoll_release_file()
1045 	struct epitem *epi, *epir = NULL;  in ep_find()  local
1050 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_find()
1051 		kcmp = ep_cmp_ffd(&ffd, &epi->ffd);  in ep_find()
1057 			epir = epi;  in ep_find()
1069 	struct epitem *epi;  in ep_find_tfd()  local
1072 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_find_tfd()
1073 		if (epi->ffd.fd == tfd) {  in ep_find_tfd()
1075 				return epi;  in ep_find_tfd()
1090 	struct epitem *epi;  in get_epoll_tfile_raw_ptr()  local
1098 	epi = ep_find_tfd(ep, tfd, toff);  in get_epoll_tfile_raw_ptr()
1099 	if (epi)  in get_epoll_tfile_raw_ptr()
1100 		file_raw = epi->ffd.file;  in get_epoll_tfile_raw_ptr()
1168 static inline bool chain_epi_lockless(struct epitem *epi)  in chain_epi_lockless()  argument
1170 	struct eventpoll *ep = epi->ep;  in chain_epi_lockless()
1173 	if (epi->next != EP_UNACTIVE_PTR)  in chain_epi_lockless()
1177 	if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)  in chain_epi_lockless()
1181 	epi->next = xchg(&ep->ovflist, epi);  in chain_epi_lockless()
1207 	struct epitem *epi = ep_item_from_wait(wait);  in ep_poll_callback()  local
1208 	struct eventpoll *ep = epi->ep;  in ep_poll_callback()
1215 	ep_set_busy_poll_napi_id(epi);  in ep_poll_callback()
1223 	if (!(epi->event.events & ~EP_PRIVATE_BITS))  in ep_poll_callback()
1232 	if (pollflags && !(pollflags & epi->event.events))  in ep_poll_callback()
1242 		if (chain_epi_lockless(epi))  in ep_poll_callback()
1243 			ep_pm_stay_awake_rcu(epi);  in ep_poll_callback()
1244 	} else if (!ep_is_linked(epi)) {  in ep_poll_callback()
1246 		if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist))  in ep_poll_callback()
1247 			ep_pm_stay_awake_rcu(epi);  in ep_poll_callback()
1255 		if ((epi->event.events & EPOLLEXCLUSIVE) &&  in ep_poll_callback()
1259 				if (epi->event.events & EPOLLIN)  in ep_poll_callback()
1263 				if (epi->event.events & EPOLLOUT)  in ep_poll_callback()
1281 		ep_poll_safewake(ep, epi);  in ep_poll_callback()
1283 	if (!(epi->event.events & EPOLLEXCLUSIVE))  in ep_poll_callback()
1312 	struct epitem *epi = ep_item_from_epqueue(pt);  in ep_ptable_queue_proc()  local
1315 	if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) {  in ep_ptable_queue_proc()
1318 		pwq->base = epi;  in ep_ptable_queue_proc()
1319 		if (epi->event.events & EPOLLEXCLUSIVE)  in ep_ptable_queue_proc()
1323 		list_add_tail(&pwq->llink, &epi->pwqlist);  in ep_ptable_queue_proc()
1324 		epi->nwait++;  in ep_ptable_queue_proc()
1327 		epi->nwait = -1;  in ep_ptable_queue_proc()
1331 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)  in ep_rbtree_insert()  argument
1341 		kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);  in ep_rbtree_insert()
1348 	rb_link_node(&epi->rbn, parent, p);  in ep_rbtree_insert()
1349 	rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);  in ep_rbtree_insert()
1393 	struct epitem *epi;  in reverse_path_check_proc()  local
1397 	list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) {  in reverse_path_check_proc()
1398 		child_file = epi->ep->file;  in reverse_path_check_proc()
1449 static int ep_create_wakeup_source(struct epitem *epi)  in ep_create_wakeup_source()  argument
1454 	if (!epi->ep->ws) {  in ep_create_wakeup_source()
1455 		epi->ep->ws = wakeup_source_register(NULL, "eventpoll");  in ep_create_wakeup_source()
1456 		if (!epi->ep->ws)  in ep_create_wakeup_source()
1460 	take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);  in ep_create_wakeup_source()
1466 	rcu_assign_pointer(epi->ws, ws);  in ep_create_wakeup_source()
1472 static noinline void ep_destroy_wakeup_source(struct epitem *epi)  in ep_destroy_wakeup_source()  argument
1474 	struct wakeup_source *ws = ep_wakeup_source(epi);  in ep_destroy_wakeup_source()
1476 	RCU_INIT_POINTER(epi->ws, NULL);  in ep_destroy_wakeup_source()
1496 	struct epitem *epi;  in ep_insert()  local
1504 	if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))  in ep_insert()
1508 	INIT_LIST_HEAD(&epi->rdllink);  in ep_insert()
1509 	INIT_LIST_HEAD(&epi->fllink);  in ep_insert()
1510 	INIT_LIST_HEAD(&epi->pwqlist);  in ep_insert()
1511 	epi->ep = ep;  in ep_insert()
1512 	ep_set_ffd(&epi->ffd, tfile, fd);  in ep_insert()
1513 	epi->event = *event;  in ep_insert()
1514 	epi->nwait = 0;  in ep_insert()
1515 	epi->next = EP_UNACTIVE_PTR;  in ep_insert()
1516 	if (epi->event.events & EPOLLWAKEUP) {  in ep_insert()
1517 		error = ep_create_wakeup_source(epi);  in ep_insert()
1521 		RCU_INIT_POINTER(epi->ws, NULL);  in ep_insert()
1526 	list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);  in ep_insert()
1533 	ep_rbtree_insert(ep, epi);  in ep_insert()
1541 	epq.epi = epi;  in ep_insert()
1551 	revents = ep_item_poll(epi, &epq.pt, 1);  in ep_insert()
1559 	if (epi->nwait < 0)  in ep_insert()
1566 	ep_set_busy_poll_napi_id(epi);  in ep_insert()
1569 	if (revents && !ep_is_linked(epi)) {  in ep_insert()
1570 		list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_insert()
1571 		ep_pm_stay_awake(epi);  in ep_insert()
1591 	ep_unregister_pollwait(ep, epi);  in ep_insert()
1594 	list_del_rcu(&epi->fllink);  in ep_insert()
1597 	rb_erase_cached(&epi->rbn, &ep->rbr);  in ep_insert()
1606 	if (ep_is_linked(epi))  in ep_insert()
1607 		list_del_init(&epi->rdllink);  in ep_insert()
1610 	wakeup_source_unregister(ep_wakeup_source(epi));  in ep_insert()
1613 	kmem_cache_free(epi_cache, epi);  in ep_insert()
1622 static int ep_modify(struct eventpoll *ep, struct epitem *epi,  in ep_modify()  argument
1637 	epi->event.events = event->events; /* need barrier below */  in ep_modify()
1638 	epi->event.data = event->data; /* protected by mtx */  in ep_modify()
1639 	if (epi->event.events & EPOLLWAKEUP) {  in ep_modify()
1640 		if (!ep_has_wakeup_source(epi))  in ep_modify()
1641 			ep_create_wakeup_source(epi);  in ep_modify()
1642 	} else if (ep_has_wakeup_source(epi)) {  in ep_modify()
1643 		ep_destroy_wakeup_source(epi);  in ep_modify()
1672 	if (ep_item_poll(epi, &pt, 1)) {  in ep_modify()
1674 		if (!ep_is_linked(epi)) {  in ep_modify()
1675 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_modify()
1676 			ep_pm_stay_awake(epi);  in ep_modify()
1699 	struct epitem *epi, *tmp;  in ep_send_events_proc()  local
1714 	list_for_each_entry_safe(epi, tmp, head, rdllink) {  in ep_send_events_proc()
1727 		ws = ep_wakeup_source(epi);  in ep_send_events_proc()
1734 		list_del_init(&epi->rdllink);  in ep_send_events_proc()
1742 		revents = ep_item_poll(epi, &pt, 1);  in ep_send_events_proc()
1747 		    __put_user(epi->event.data, &uevent->data)) {  in ep_send_events_proc()
1748 			list_add(&epi->rdllink, head);  in ep_send_events_proc()
1749 			ep_pm_stay_awake(epi);  in ep_send_events_proc()
1756 		if (epi->event.events & EPOLLONESHOT)  in ep_send_events_proc()
1757 			epi->event.events &= EP_PRIVATE_BITS;  in ep_send_events_proc()
1758 		else if (!(epi->event.events & EPOLLET)) {  in ep_send_events_proc()
1770 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_send_events_proc()
1771 			ep_pm_stay_awake(epi);  in ep_send_events_proc()
1970 	struct epitem *epi;  in ep_loop_check_proc()  local
1975 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_loop_check_proc()
1976 		if (unlikely(is_file_epoll(epi->ffd.file))) {  in ep_loop_check_proc()
1977 			ep_tovisit = epi->ffd.file->private_data;  in ep_loop_check_proc()
1981 					ep_loop_check_proc, epi->ffd.file,  in ep_loop_check_proc()
1994 			if (list_empty(&epi->ffd.file->f_tfile_llink)) {  in ep_loop_check_proc()
1995 				if (get_file_rcu(epi->ffd.file))  in ep_loop_check_proc()
1996 					list_add(&epi->ffd.file->f_tfile_llink,  in ep_loop_check_proc()
2115 	struct epitem *epi;  in do_epoll_ctl()  local
2221 	epi = ep_find(ep, tf.file, fd);  in do_epoll_ctl()
2226 		if (!epi) {  in do_epoll_ctl()
2233 		if (epi)  in do_epoll_ctl()
2234 			error = ep_remove(ep, epi);  in do_epoll_ctl()
2239 		if (epi) {  in do_epoll_ctl()
2240 			if (!(epi->event.events & EPOLLEXCLUSIVE)) {  in do_epoll_ctl()
2242 				error = ep_modify(ep, epi, epds);  in do_epoll_ctl()