214 	struct epitem *epi;  member
302 	return container_of(p, struct ep_pqueue, pt)->epi;  in ep_item_from_epqueue()
445 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)  in ep_unregister_pollwait()  argument
447 	struct list_head *lsthead = &epi->pwqlist;  in ep_unregister_pollwait()
477 	struct epitem *epi, *nepi;  in ep_scan_ready_list()  local
510 	for (nepi = ep->ovflist; (epi = nepi) != NULL;  in ep_scan_ready_list()
511 	     nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {  in ep_scan_ready_list()
518 		if (!ep_is_linked(&epi->rdllink))  in ep_scan_ready_list()
519 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_scan_ready_list()
558 static int ep_remove(struct eventpoll *ep, struct epitem *epi)  in ep_remove()  argument
561 	struct file *file = epi->ffd.file;  in ep_remove()
571 	ep_unregister_pollwait(ep, epi);  in ep_remove()
575 	if (ep_is_linked(&epi->fllink))  in ep_remove()
576 		list_del_init(&epi->fllink);  in ep_remove()
579 	rb_erase(&epi->rbn, &ep->rbr);  in ep_remove()
582 	if (ep_is_linked(&epi->rdllink))  in ep_remove()
583 		list_del_init(&epi->rdllink);  in ep_remove()
587 	kmem_cache_free(epi_cache, epi);  in ep_remove()
597 	struct epitem *epi;  in ep_free()  local
617 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
619 		ep_unregister_pollwait(ep, epi);  in ep_free()
629 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
630 		ep_remove(ep, epi);  in ep_free()
652 	struct epitem *epi, *tmp;  in ep_read_events_proc()  local
654 	list_for_each_entry_safe(epi, tmp, head, rdllink) {  in ep_read_events_proc()
655 		if (epi->ffd.file->f_op->poll(epi->ffd.file, NULL) &  in ep_read_events_proc()
656 		    epi->event.events)  in ep_read_events_proc()
664 			list_del_init(&epi->rdllink);  in ep_read_events_proc()
718 	struct epitem *epi;  in eventpoll_release_file()  local
736 		epi = list_first_entry(lsthead, struct epitem, fllink);  in eventpoll_release_file()
738 		ep = epi->ep;  in eventpoll_release_file()
739 		list_del_init(&epi->fllink);  in eventpoll_release_file()
741 		ep_remove(ep, epi);  in eventpoll_release_file()
787 	struct epitem *epi, *epir = NULL;  in ep_find()  local
792 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_find()
793 		kcmp = ep_cmp_ffd(&ffd, &epi->ffd);  in ep_find()
799 			epir = epi;  in ep_find()
816 	struct epitem *epi = ep_item_from_wait(wait);  in ep_poll_callback()  local
817 	struct eventpoll *ep = epi->ep;  in ep_poll_callback()
827 	if (!(epi->event.events & ~EP_PRIVATE_BITS))  in ep_poll_callback()
836 	if (key && !((unsigned long) key & epi->event.events))  in ep_poll_callback()
846 		if (epi->next == EP_UNACTIVE_PTR) {  in ep_poll_callback()
847 			epi->next = ep->ovflist;  in ep_poll_callback()
848 			ep->ovflist = epi;  in ep_poll_callback()
854 	if (!ep_is_linked(&epi->rdllink))  in ep_poll_callback()
855 		list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_poll_callback()
883 	struct epitem *epi = ep_item_from_epqueue(pt);  in ep_ptable_queue_proc()  local
886 	if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) {  in ep_ptable_queue_proc()
889 		pwq->base = epi;  in ep_ptable_queue_proc()
891 		list_add_tail(&pwq->llink, &epi->pwqlist);  in ep_ptable_queue_proc()
892 		epi->nwait++;  in ep_ptable_queue_proc()
895 		epi->nwait = -1;  in ep_ptable_queue_proc()
899 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)  in ep_rbtree_insert()  argument
908 		kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);  in ep_rbtree_insert()
914 	rb_link_node(&epi->rbn, parent, p);  in ep_rbtree_insert()
915 	rb_insert_color(&epi->rbn, &ep->rbr);  in ep_rbtree_insert()
927 	struct epitem *epi;  in ep_insert()  local
933 	if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))  in ep_insert()
937 	INIT_LIST_HEAD(&epi->rdllink);  in ep_insert()
938 	INIT_LIST_HEAD(&epi->fllink);  in ep_insert()
939 	INIT_LIST_HEAD(&epi->pwqlist);  in ep_insert()
940 	epi->ep = ep;  in ep_insert()
941 	ep_set_ffd(&epi->ffd, tfile, fd);  in ep_insert()
942 	epi->event = *event;  in ep_insert()
943 	epi->nwait = 0;  in ep_insert()
944 	epi->next = EP_UNACTIVE_PTR;  in ep_insert()
947 	epq.epi = epi;  in ep_insert()
965 	if (epi->nwait < 0)  in ep_insert()
970 	list_add_tail(&epi->fllink, &tfile->f_ep_links);  in ep_insert()
977 	ep_rbtree_insert(ep, epi);  in ep_insert()
983 	if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) {  in ep_insert()
984 		list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_insert()
1004 	ep_unregister_pollwait(ep, epi);  in ep_insert()
1013 	if (ep_is_linked(&epi->rdllink))  in ep_insert()
1014 		list_del_init(&epi->rdllink);  in ep_insert()
1017 	kmem_cache_free(epi_cache, epi);  in ep_insert()
1026 static int ep_modify(struct eventpoll *ep, struct epitem *epi, struct epoll_event *event)  in ep_modify()  argument
1036 	epi->event.events = event->events;  in ep_modify()
1037 	epi->event.data = event->data; /* protected by mtx */  in ep_modify()
1043 	revents = epi->ffd.file->f_op->poll(epi->ffd.file, NULL);  in ep_modify()
1051 		if (!ep_is_linked(&epi->rdllink)) {  in ep_modify()
1052 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_modify()
1076 	struct epitem *epi;  in ep_send_events_proc()  local
1086 		epi = list_first_entry(head, struct epitem, rdllink);  in ep_send_events_proc()
1088 		list_del_init(&epi->rdllink);  in ep_send_events_proc()
1090 		revents = epi->ffd.file->f_op->poll(epi->ffd.file, NULL) &  in ep_send_events_proc()
1091 			epi->event.events;  in ep_send_events_proc()
1101 			    __put_user(epi->event.data, &uevent->data)) {  in ep_send_events_proc()
1102 				list_add(&epi->rdllink, head);  in ep_send_events_proc()
1107 			if (epi->event.events & EPOLLONESHOT)  in ep_send_events_proc()
1108 				epi->event.events &= EP_PRIVATE_BITS;  in ep_send_events_proc()
1109 			else if (!(epi->event.events & EPOLLET)) {  in ep_send_events_proc()
1121 				list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_send_events_proc()
1268 	struct epitem *epi;  in ep_loop_check_proc()  local
1272 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_loop_check_proc()
1273 		if (unlikely(is_file_epoll(epi->ffd.file))) {  in ep_loop_check_proc()
1275 					       ep_loop_check_proc, epi->ffd.file,  in ep_loop_check_proc()
1276 					       epi->ffd.file->private_data, current);  in ep_loop_check_proc()
1354 	struct epitem *epi;  in SYSCALL_DEFINE4()  local
1419 	epi = ep_find(ep, tfile, fd);  in SYSCALL_DEFINE4()
1424 		if (!epi) {  in SYSCALL_DEFINE4()
1431 		if (epi)  in SYSCALL_DEFINE4()
1432 			error = ep_remove(ep, epi);  in SYSCALL_DEFINE4()
1437 		if (epi) {  in SYSCALL_DEFINE4()
1439 			error = ep_modify(ep, epi, &epds);  in SYSCALL_DEFINE4()