40 	struct watch_queue *wqueue = (struct watch_queue *)buf->private;  in watch_queue_pipe_buf_release()  local
56 	set_bit(bit, wqueue->notes_bitmap);  in watch_queue_pipe_buf_release()
72 static bool post_one_notification(struct watch_queue *wqueue,  in post_one_notification()  argument
76 	struct pipe_inode_info *pipe = wqueue->pipe;  in post_one_notification()
87 	if (wqueue->defunct)  in post_one_notification()
96 	note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);  in post_one_notification()
97 	if (note >= wqueue->nr_notes)  in post_one_notification()
100 	page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE];  in post_one_notification()
110 	buf->private = (unsigned long)wqueue;  in post_one_notification()
117 	if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {  in post_one_notification()
181 	struct watch_queue *wqueue;  in __post_watch_notification()  local
197 		wqueue = rcu_dereference(watch->queue);  in __post_watch_notification()
198 		wf = rcu_dereference(wqueue->filter);  in __post_watch_notification()
205 		post_one_notification(wqueue, n);  in __post_watch_notification()
218 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_size()  local
225 	if (!wqueue)  in watch_queue_set_size()
227 	if (wqueue->notes)  in watch_queue_set_size()
268 	wqueue->notes = pages;  in watch_queue_set_size()
269 	wqueue->notes_bitmap = bitmap;  in watch_queue_set_size()
270 	wqueue->nr_pages = nr_pages;  in watch_queue_set_size()
271 	wqueue->nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE;  in watch_queue_set_size()
293 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_filter()  local
296 	if (!wqueue)  in watch_queue_set_filter()
353 	wfilter = rcu_replace_pointer(wqueue->filter, wfilter,  in watch_queue_set_filter()
367 	struct watch_queue *wqueue =  in __put_watch_queue()  local
372 	for (i = 0; i < wqueue->nr_pages; i++)  in __put_watch_queue()
373 		__free_page(wqueue->notes[i]);  in __put_watch_queue()
375 	wfilter = rcu_access_pointer(wqueue->filter);  in __put_watch_queue()
378 	kfree_rcu(wqueue, rcu);  in __put_watch_queue()
385 void put_watch_queue(struct watch_queue *wqueue)  in put_watch_queue()  argument
387 	kref_put(&wqueue->usage, __put_watch_queue);  in put_watch_queue()
422 void init_watch(struct watch *watch, struct watch_queue *wqueue)  in init_watch()  argument
427 	rcu_assign_pointer(watch->queue, wqueue);  in init_watch()
444 	struct watch_queue *wqueue = rcu_access_pointer(watch->queue);  in add_watch_to_object()  local
449 		if (wqueue == wq && watch->id == w->id)  in add_watch_to_object()
463 	spin_lock_bh(&wqueue->lock);  in add_watch_to_object()
464 	kref_get(&wqueue->usage);  in add_watch_to_object()
466 	hlist_add_head(&watch->queue_node, &wqueue->watches);  in add_watch_to_object()
467 	spin_unlock_bh(&wqueue->lock);  in add_watch_to_object()
488 	struct watch_queue *wqueue;  in remove_watch_from_object()  local
519 	wqueue = rcu_dereference(watch->queue);  in remove_watch_from_object()
524 	if (wqueue) {  in remove_watch_from_object()
525 		post_one_notification(wqueue, &n.watch);  in remove_watch_from_object()
527 		spin_lock_bh(&wqueue->lock);  in remove_watch_from_object()
534 		spin_unlock_bh(&wqueue->lock);  in remove_watch_from_object()
560 void watch_queue_clear(struct watch_queue *wqueue)  in watch_queue_clear()  argument
567 	spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
570 	wqueue->defunct = true;  in watch_queue_clear()
572 	while (!hlist_empty(&wqueue->watches)) {  in watch_queue_clear()
573 		watch = hlist_entry(wqueue->watches.first, struct watch, queue_node);  in watch_queue_clear()
576 		spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
614 		spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
617 	spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
628 	struct watch_queue *wqueue = ERR_PTR(-EINVAL);  in get_watch_queue()  local
635 			wqueue = pipe->watch_queue;  in get_watch_queue()
636 			kref_get(&wqueue->usage);  in get_watch_queue()
641 	return wqueue;  in get_watch_queue()
650 	struct watch_queue *wqueue;  in watch_queue_init()  local
652 	wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL);  in watch_queue_init()
653 	if (!wqueue)  in watch_queue_init()
656 	wqueue->pipe = pipe;  in watch_queue_init()
657 	kref_init(&wqueue->usage);  in watch_queue_init()
658 	spin_lock_init(&wqueue->lock);  in watch_queue_init()
659 	INIT_HLIST_HEAD(&wqueue->watches);  in watch_queue_init()
661 	pipe->watch_queue = wqueue;  in watch_queue_init()