43 static inline bool lock_wqueue(struct watch_queue *wqueue)  in lock_wqueue()  argument
45 	spin_lock_bh(&wqueue->lock);  in lock_wqueue()
46 	if (unlikely(wqueue->defunct)) {  in lock_wqueue()
47 		spin_unlock_bh(&wqueue->lock);  in lock_wqueue()
53 static inline void unlock_wqueue(struct watch_queue *wqueue)  in unlock_wqueue()  argument
55 	spin_unlock_bh(&wqueue->lock);  in unlock_wqueue()
61 	struct watch_queue *wqueue = (struct watch_queue *)buf->private;  in watch_queue_pipe_buf_release()  local
77 	set_bit(bit, wqueue->notes_bitmap);  in watch_queue_pipe_buf_release()
98 static bool post_one_notification(struct watch_queue *wqueue,  in post_one_notification()  argument
102 	struct pipe_inode_info *pipe = wqueue->pipe;  in post_one_notification()
119 	note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);  in post_one_notification()
120 	if (note >= wqueue->nr_notes)  in post_one_notification()
123 	page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE];  in post_one_notification()
133 	buf->private = (unsigned long)wqueue;  in post_one_notification()
140 	if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {  in post_one_notification()
204 	struct watch_queue *wqueue;  in __post_watch_notification()  local
220 		wqueue = rcu_dereference(watch->queue);  in __post_watch_notification()
221 		wf = rcu_dereference(wqueue->filter);  in __post_watch_notification()
228 		if (lock_wqueue(wqueue)) {  in __post_watch_notification()
229 			post_one_notification(wqueue, n);  in __post_watch_notification()
230 			unlock_wqueue(wqueue);  in __post_watch_notification()
244 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_size()  local
250 	if (!wqueue)  in watch_queue_set_size()
252 	if (wqueue->notes)  in watch_queue_set_size()
292 	wqueue->notes = pages;  in watch_queue_set_size()
293 	wqueue->notes_bitmap = bitmap;  in watch_queue_set_size()
294 	wqueue->nr_pages = nr_pages;  in watch_queue_set_size()
295 	wqueue->nr_notes = nr_notes;  in watch_queue_set_size()
317 	struct watch_queue *wqueue = pipe->watch_queue;  in watch_queue_set_filter()  local
320 	if (!wqueue)  in watch_queue_set_filter()
377 	wfilter = rcu_replace_pointer(wqueue->filter, wfilter,  in watch_queue_set_filter()
391 	struct watch_queue *wqueue =  in __put_watch_queue()  local
396 	for (i = 0; i < wqueue->nr_pages; i++)  in __put_watch_queue()
397 		__free_page(wqueue->notes[i]);  in __put_watch_queue()
398 	kfree(wqueue->notes);  in __put_watch_queue()
399 	bitmap_free(wqueue->notes_bitmap);  in __put_watch_queue()
401 	wfilter = rcu_access_pointer(wqueue->filter);  in __put_watch_queue()
404 	kfree_rcu(wqueue, rcu);  in __put_watch_queue()
411 void put_watch_queue(struct watch_queue *wqueue)  in put_watch_queue()  argument
413 	kref_put(&wqueue->usage, __put_watch_queue);  in put_watch_queue()
449 void init_watch(struct watch *watch, struct watch_queue *wqueue)  in init_watch()  argument
454 	rcu_assign_pointer(watch->queue, wqueue);  in init_watch()
457 static int add_one_watch(struct watch *watch, struct watch_list *wlist, struct watch_queue *wqueue)  in add_one_watch()  argument
464 		if (wqueue == wq && watch->id == w->id)  in add_one_watch()
477 	kref_get(&wqueue->usage);  in add_one_watch()
479 	hlist_add_head(&watch->queue_node, &wqueue->watches);  in add_one_watch()
498 	struct watch_queue *wqueue;  in add_watch_to_object()  local
503 	wqueue = rcu_access_pointer(watch->queue);  in add_watch_to_object()
504 	if (lock_wqueue(wqueue)) {  in add_watch_to_object()
506 		ret = add_one_watch(watch, wlist, wqueue);  in add_watch_to_object()
508 		unlock_wqueue(wqueue);  in add_watch_to_object()
530 	struct watch_queue *wqueue;  in remove_watch_from_object()  local
561 	wqueue = rcu_dereference(watch->queue);  in remove_watch_from_object()
563 	if (lock_wqueue(wqueue)) {  in remove_watch_from_object()
564 		post_one_notification(wqueue, &n.watch);  in remove_watch_from_object()
571 		unlock_wqueue(wqueue);  in remove_watch_from_object()
597 void watch_queue_clear(struct watch_queue *wqueue)  in watch_queue_clear()  argument
604 	spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
607 	wqueue->defunct = true;  in watch_queue_clear()
609 	while (!hlist_empty(&wqueue->watches)) {  in watch_queue_clear()
610 		watch = hlist_entry(wqueue->watches.first, struct watch, queue_node);  in watch_queue_clear()
613 		spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
651 		spin_lock_bh(&wqueue->lock);  in watch_queue_clear()
654 	spin_unlock_bh(&wqueue->lock);  in watch_queue_clear()
665 	struct watch_queue *wqueue = ERR_PTR(-EINVAL);  in get_watch_queue()  local
672 			wqueue = pipe->watch_queue;  in get_watch_queue()
673 			kref_get(&wqueue->usage);  in get_watch_queue()
678 	return wqueue;  in get_watch_queue()
687 	struct watch_queue *wqueue;  in watch_queue_init()  local
689 	wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL);  in watch_queue_init()
690 	if (!wqueue)  in watch_queue_init()
693 	wqueue->pipe = pipe;  in watch_queue_init()
694 	kref_init(&wqueue->usage);  in watch_queue_init()
695 	spin_lock_init(&wqueue->lock);  in watch_queue_init()
696 	INIT_HLIST_HEAD(&wqueue->watches);  in watch_queue_init()
698 	pipe->watch_queue = wqueue;  in watch_queue_init()