94 void fsnotify_get_mark(struct fsnotify_mark *mark)  in fsnotify_get_mark()  argument
96 	WARN_ON_ONCE(!refcount_read(&mark->refcnt));  in fsnotify_get_mark()
97 	refcount_inc(&mark->refcnt);  in fsnotify_get_mark()
122 	struct fsnotify_mark *mark;  in __fsnotify_recalc_mask()  local
128 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in __fsnotify_recalc_mask()
129 		if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)  in __fsnotify_recalc_mask()
130 			new_mask |= mark->mask;  in __fsnotify_recalc_mask()
199 static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)  in fsnotify_final_mark_destroy()  argument
201 	struct fsnotify_group *group = mark->group;  in fsnotify_final_mark_destroy()
205 	group->ops->free_mark(mark);  in fsnotify_final_mark_destroy()
227 void fsnotify_put_mark(struct fsnotify_mark *mark)  in fsnotify_put_mark()  argument
229 	struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);  in fsnotify_put_mark()
236 		if (refcount_dec_and_test(&mark->refcnt))  in fsnotify_put_mark()
237 			fsnotify_final_mark_destroy(mark);  in fsnotify_put_mark()
245 	if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))  in fsnotify_put_mark()
248 	hlist_del_init_rcu(&mark->obj_list);  in fsnotify_put_mark()
255 	WRITE_ONCE(mark->connector, NULL);  in fsnotify_put_mark()
274 	list_add(&mark->g_list, &destroy_list);  in fsnotify_put_mark()
288 static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)  in fsnotify_get_mark_safe()  argument
290 	if (!mark)  in fsnotify_get_mark_safe()
293 	if (refcount_inc_not_zero(&mark->refcnt)) {  in fsnotify_get_mark_safe()
294 		spin_lock(&mark->lock);  in fsnotify_get_mark_safe()
295 		if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {  in fsnotify_get_mark_safe()
297 			atomic_inc(&mark->group->user_waits);  in fsnotify_get_mark_safe()
298 			spin_unlock(&mark->lock);  in fsnotify_get_mark_safe()
301 		spin_unlock(&mark->lock);  in fsnotify_get_mark_safe()
302 		fsnotify_put_mark(mark);  in fsnotify_get_mark_safe()
312 static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)  in fsnotify_put_mark_wake()  argument
314 	if (mark) {  in fsnotify_put_mark_wake()
315 		struct fsnotify_group *group = mark->group;  in fsnotify_put_mark_wake()
317 		fsnotify_put_mark(mark);  in fsnotify_put_mark_wake()
371 void fsnotify_detach_mark(struct fsnotify_mark *mark)  in fsnotify_detach_mark()  argument
373 	struct fsnotify_group *group = mark->group;  in fsnotify_detach_mark()
377 		     refcount_read(&mark->refcnt) < 1 +  in fsnotify_detach_mark()
378 			!!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));  in fsnotify_detach_mark()
380 	spin_lock(&mark->lock);  in fsnotify_detach_mark()
382 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {  in fsnotify_detach_mark()
383 		spin_unlock(&mark->lock);  in fsnotify_detach_mark()
386 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;  in fsnotify_detach_mark()
387 	list_del_init(&mark->g_list);  in fsnotify_detach_mark()
388 	spin_unlock(&mark->lock);  in fsnotify_detach_mark()
393 	fsnotify_put_mark(mark);  in fsnotify_detach_mark()
404 void fsnotify_free_mark(struct fsnotify_mark *mark)  in fsnotify_free_mark()  argument
406 	struct fsnotify_group *group = mark->group;  in fsnotify_free_mark()
408 	spin_lock(&mark->lock);  in fsnotify_free_mark()
410 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {  in fsnotify_free_mark()
411 		spin_unlock(&mark->lock);  in fsnotify_free_mark()
414 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;  in fsnotify_free_mark()
415 	spin_unlock(&mark->lock);  in fsnotify_free_mark()
423 		group->ops->freeing_mark(mark, group);  in fsnotify_free_mark()
426 void fsnotify_destroy_mark(struct fsnotify_mark *mark,  in fsnotify_destroy_mark()  argument
430 	fsnotify_detach_mark(mark);  in fsnotify_destroy_mark()
432 	fsnotify_free_mark(mark);  in fsnotify_destroy_mark()
543 static int fsnotify_add_mark_list(struct fsnotify_mark *mark,  in fsnotify_add_mark_list()  argument
560 	spin_lock(&mark->lock);  in fsnotify_add_mark_list()
563 		spin_unlock(&mark->lock);  in fsnotify_add_mark_list()
593 		hlist_add_head_rcu(&mark->obj_list, &conn->list);  in fsnotify_add_mark_list()
601 		if ((lmark->group == mark->group) &&  in fsnotify_add_mark_list()
608 		cmp = fsnotify_compare_groups(lmark->group, mark->group);  in fsnotify_add_mark_list()
610 			hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);  in fsnotify_add_mark_list()
617 	hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);  in fsnotify_add_mark_list()
624 	WRITE_ONCE(mark->connector, conn);  in fsnotify_add_mark_list()
627 	spin_unlock(&mark->lock);  in fsnotify_add_mark_list()
636 int fsnotify_add_mark_locked(struct fsnotify_mark *mark,  in fsnotify_add_mark_locked()  argument
640 	struct fsnotify_group *group = mark->group;  in fsnotify_add_mark_locked()
651 	spin_lock(&mark->lock);  in fsnotify_add_mark_locked()
652 	mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;  in fsnotify_add_mark_locked()
654 	list_add(&mark->g_list, &group->marks_list);  in fsnotify_add_mark_locked()
656 	fsnotify_get_mark(mark); /* for g_list */  in fsnotify_add_mark_locked()
657 	spin_unlock(&mark->lock);  in fsnotify_add_mark_locked()
659 	ret = fsnotify_add_mark_list(mark, connp, type, allow_dups, fsid);  in fsnotify_add_mark_locked()
663 	if (mark->mask)  in fsnotify_add_mark_locked()
664 		fsnotify_recalc_mask(mark->connector);  in fsnotify_add_mark_locked()
668 	spin_lock(&mark->lock);  in fsnotify_add_mark_locked()
669 	mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |  in fsnotify_add_mark_locked()
671 	list_del_init(&mark->g_list);  in fsnotify_add_mark_locked()
672 	spin_unlock(&mark->lock);  in fsnotify_add_mark_locked()
675 	fsnotify_put_mark(mark);  in fsnotify_add_mark_locked()
679 int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp,  in fsnotify_add_mark()  argument
683 	struct fsnotify_group *group = mark->group;  in fsnotify_add_mark()
686 	ret = fsnotify_add_mark_locked(mark, connp, type, allow_dups, fsid);  in fsnotify_add_mark()
700 	struct fsnotify_mark *mark;  in fsnotify_find_mark()  local
706 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_find_mark()
707 		if (mark->group == group &&  in fsnotify_find_mark()
708 		    (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {  in fsnotify_find_mark()
709 			fsnotify_get_mark(mark);  in fsnotify_find_mark()
711 			return mark;  in fsnotify_find_mark()
723 	struct fsnotify_mark *lmark, *mark;  in fsnotify_clear_marks_by_group()  local
742 	list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {  in fsnotify_clear_marks_by_group()
743 		if ((1U << mark->connector->type) & type_mask)  in fsnotify_clear_marks_by_group()
744 			list_move(&mark->g_list, &to_free);  in fsnotify_clear_marks_by_group()
755 		mark = list_first_entry(head, struct fsnotify_mark, g_list);  in fsnotify_clear_marks_by_group()
756 		fsnotify_get_mark(mark);  in fsnotify_clear_marks_by_group()
757 		fsnotify_detach_mark(mark);  in fsnotify_clear_marks_by_group()
759 		fsnotify_free_mark(mark);  in fsnotify_clear_marks_by_group()
760 		fsnotify_put_mark(mark);  in fsnotify_clear_marks_by_group()
768 	struct fsnotify_mark *mark, *old_mark = NULL;  in fsnotify_destroy_marks()  local
782 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_destroy_marks()
783 		fsnotify_get_mark(mark);  in fsnotify_destroy_marks()
787 		old_mark = mark;  in fsnotify_destroy_marks()
788 		fsnotify_destroy_mark(mark, mark->group);  in fsnotify_destroy_marks()
806 void fsnotify_init_mark(struct fsnotify_mark *mark,  in fsnotify_init_mark()  argument
809 	memset(mark, 0, sizeof(*mark));  in fsnotify_init_mark()
810 	spin_lock_init(&mark->lock);  in fsnotify_init_mark()
811 	refcount_set(&mark->refcnt, 1);  in fsnotify_init_mark()
813 	mark->group = group;  in fsnotify_init_mark()
814 	WRITE_ONCE(mark->connector, NULL);  in fsnotify_init_mark()
824 	struct fsnotify_mark *mark, *next;  in fsnotify_mark_destroy_workfn()  local
834 	list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {  in fsnotify_mark_destroy_workfn()
835 		list_del_init(&mark->g_list);  in fsnotify_mark_destroy_workfn()
836 		fsnotify_final_mark_destroy(mark);  in fsnotify_mark_destroy_workfn()