106 void fsnotify_get_mark(struct fsnotify_mark *mark)  in fsnotify_get_mark()  argument
108 	WARN_ON_ONCE(!refcount_read(&mark->refcnt));  in fsnotify_get_mark()
109 	refcount_inc(&mark->refcnt);  in fsnotify_get_mark()
132 	struct fsnotify_mark *mark;  in __fsnotify_recalc_mask()  local
138 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in __fsnotify_recalc_mask()
139 		if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)  in __fsnotify_recalc_mask()
140 			new_mask |= mark->mask;  in __fsnotify_recalc_mask()
204 static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)  in fsnotify_final_mark_destroy()  argument
206 	struct fsnotify_group *group = mark->group;  in fsnotify_final_mark_destroy()
210 	group->ops->free_mark(mark);  in fsnotify_final_mark_destroy()
214 void fsnotify_put_mark(struct fsnotify_mark *mark)  in fsnotify_put_mark()  argument
221 	if (!mark->connector) {  in fsnotify_put_mark()
222 		if (refcount_dec_and_test(&mark->refcnt))  in fsnotify_put_mark()
223 			fsnotify_final_mark_destroy(mark);  in fsnotify_put_mark()
231 	if (!refcount_dec_and_lock(&mark->refcnt, &mark->connector->lock))  in fsnotify_put_mark()
234 	conn = mark->connector;  in fsnotify_put_mark()
235 	hlist_del_init_rcu(&mark->obj_list);  in fsnotify_put_mark()
242 	mark->connector = NULL;  in fsnotify_put_mark()
261 	list_add(&mark->g_list, &destroy_list);  in fsnotify_put_mark()
274 static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)  in fsnotify_get_mark_safe()  argument
276 	if (!mark)  in fsnotify_get_mark_safe()
279 	if (refcount_inc_not_zero(&mark->refcnt)) {  in fsnotify_get_mark_safe()
280 		spin_lock(&mark->lock);  in fsnotify_get_mark_safe()
281 		if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {  in fsnotify_get_mark_safe()
283 			atomic_inc(&mark->group->user_waits);  in fsnotify_get_mark_safe()
284 			spin_unlock(&mark->lock);  in fsnotify_get_mark_safe()
287 		spin_unlock(&mark->lock);  in fsnotify_get_mark_safe()
288 		fsnotify_put_mark(mark);  in fsnotify_get_mark_safe()
298 static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)  in fsnotify_put_mark_wake()  argument
300 	if (mark) {  in fsnotify_put_mark_wake()
301 		struct fsnotify_group *group = mark->group;  in fsnotify_put_mark_wake()
303 		fsnotify_put_mark(mark);  in fsnotify_put_mark_wake()
357 void fsnotify_detach_mark(struct fsnotify_mark *mark)  in fsnotify_detach_mark()  argument
359 	struct fsnotify_group *group = mark->group;  in fsnotify_detach_mark()
363 		     refcount_read(&mark->refcnt) < 1 +  in fsnotify_detach_mark()
364 			!!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));  in fsnotify_detach_mark()
366 	spin_lock(&mark->lock);  in fsnotify_detach_mark()
368 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {  in fsnotify_detach_mark()
369 		spin_unlock(&mark->lock);  in fsnotify_detach_mark()
372 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;  in fsnotify_detach_mark()
373 	list_del_init(&mark->g_list);  in fsnotify_detach_mark()
374 	spin_unlock(&mark->lock);  in fsnotify_detach_mark()
379 	fsnotify_put_mark(mark);  in fsnotify_detach_mark()
390 void fsnotify_free_mark(struct fsnotify_mark *mark)  in fsnotify_free_mark()  argument
392 	struct fsnotify_group *group = mark->group;  in fsnotify_free_mark()
394 	spin_lock(&mark->lock);  in fsnotify_free_mark()
396 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {  in fsnotify_free_mark()
397 		spin_unlock(&mark->lock);  in fsnotify_free_mark()
400 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;  in fsnotify_free_mark()
401 	spin_unlock(&mark->lock);  in fsnotify_free_mark()
409 		group->ops->freeing_mark(mark, group);  in fsnotify_free_mark()
412 void fsnotify_destroy_mark(struct fsnotify_mark *mark,  in fsnotify_destroy_mark()  argument
416 	fsnotify_detach_mark(mark);  in fsnotify_destroy_mark()
418 	fsnotify_free_mark(mark);  in fsnotify_destroy_mark()
519 static int fsnotify_add_mark_list(struct fsnotify_mark *mark,  in fsnotify_add_mark_list()  argument
531 	spin_lock(&mark->lock);  in fsnotify_add_mark_list()
534 		spin_unlock(&mark->lock);  in fsnotify_add_mark_list()
543 		hlist_add_head_rcu(&mark->obj_list, &conn->list);  in fsnotify_add_mark_list()
551 		if ((lmark->group == mark->group) &&  in fsnotify_add_mark_list()
558 		cmp = fsnotify_compare_groups(lmark->group, mark->group);  in fsnotify_add_mark_list()
560 			hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);  in fsnotify_add_mark_list()
567 	hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);  in fsnotify_add_mark_list()
569 	mark->connector = conn;  in fsnotify_add_mark_list()
572 	spin_unlock(&mark->lock);  in fsnotify_add_mark_list()
581 int fsnotify_add_mark_locked(struct fsnotify_mark *mark,  in fsnotify_add_mark_locked()  argument
585 	struct fsnotify_group *group = mark->group;  in fsnotify_add_mark_locked()
596 	spin_lock(&mark->lock);  in fsnotify_add_mark_locked()
597 	mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;  in fsnotify_add_mark_locked()
599 	list_add(&mark->g_list, &group->marks_list);  in fsnotify_add_mark_locked()
601 	fsnotify_get_mark(mark); /* for g_list */  in fsnotify_add_mark_locked()
602 	spin_unlock(&mark->lock);  in fsnotify_add_mark_locked()
604 	ret = fsnotify_add_mark_list(mark, connp, type, allow_dups);  in fsnotify_add_mark_locked()
608 	if (mark->mask)  in fsnotify_add_mark_locked()
609 		fsnotify_recalc_mask(mark->connector);  in fsnotify_add_mark_locked()
613 	spin_lock(&mark->lock);  in fsnotify_add_mark_locked()
614 	mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |  in fsnotify_add_mark_locked()
616 	list_del_init(&mark->g_list);  in fsnotify_add_mark_locked()
617 	spin_unlock(&mark->lock);  in fsnotify_add_mark_locked()
620 	fsnotify_put_mark(mark);  in fsnotify_add_mark_locked()
624 int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp,  in fsnotify_add_mark()  argument
628 	struct fsnotify_group *group = mark->group;  in fsnotify_add_mark()
631 	ret = fsnotify_add_mark_locked(mark, connp, type, allow_dups);  in fsnotify_add_mark()
644 	struct fsnotify_mark *mark;  in fsnotify_find_mark()  local
650 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_find_mark()
651 		if (mark->group == group &&  in fsnotify_find_mark()
652 		    (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {  in fsnotify_find_mark()
653 			fsnotify_get_mark(mark);  in fsnotify_find_mark()
655 			return mark;  in fsnotify_find_mark()
666 	struct fsnotify_mark *lmark, *mark;  in fsnotify_clear_marks_by_group()  local
685 	list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {  in fsnotify_clear_marks_by_group()
686 		if ((1U << mark->connector->type) & type_mask)  in fsnotify_clear_marks_by_group()
687 			list_move(&mark->g_list, &to_free);  in fsnotify_clear_marks_by_group()
698 		mark = list_first_entry(head, struct fsnotify_mark, g_list);  in fsnotify_clear_marks_by_group()
699 		fsnotify_get_mark(mark);  in fsnotify_clear_marks_by_group()
700 		fsnotify_detach_mark(mark);  in fsnotify_clear_marks_by_group()
702 		fsnotify_free_mark(mark);  in fsnotify_clear_marks_by_group()
703 		fsnotify_put_mark(mark);  in fsnotify_clear_marks_by_group()
711 	struct fsnotify_mark *mark, *old_mark = NULL;  in fsnotify_destroy_marks()  local
724 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_destroy_marks()
725 		fsnotify_get_mark(mark);  in fsnotify_destroy_marks()
729 		old_mark = mark;  in fsnotify_destroy_marks()
730 		fsnotify_destroy_mark(mark, mark->group);  in fsnotify_destroy_marks()
748 void fsnotify_init_mark(struct fsnotify_mark *mark,  in fsnotify_init_mark()  argument
751 	memset(mark, 0, sizeof(*mark));  in fsnotify_init_mark()
752 	spin_lock_init(&mark->lock);  in fsnotify_init_mark()
753 	refcount_set(&mark->refcnt, 1);  in fsnotify_init_mark()
755 	mark->group = group;  in fsnotify_init_mark()
764 	struct fsnotify_mark *mark, *next;  in fsnotify_mark_destroy_workfn()  local
774 	list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {  in fsnotify_mark_destroy_workfn()
775 		list_del_init(&mark->g_list);  in fsnotify_mark_destroy_workfn()
776 		fsnotify_final_mark_destroy(mark);  in fsnotify_mark_destroy_workfn()