100 static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn)  in fsnotify_conn_mask_p()  argument
102 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)  in fsnotify_conn_mask_p()
103 		return &fsnotify_conn_inode(conn)->i_fsnotify_mask;  in fsnotify_conn_mask_p()
104 	else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)  in fsnotify_conn_mask_p()
105 		return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask;  in fsnotify_conn_mask_p()
106 	else if (conn->type == FSNOTIFY_OBJ_TYPE_SB)  in fsnotify_conn_mask_p()
107 		return &fsnotify_conn_sb(conn)->s_fsnotify_mask;  in fsnotify_conn_mask_p()
111 __u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn)  in fsnotify_conn_mask()  argument
113 	if (WARN_ON(!fsnotify_valid_obj_type(conn->type)))  in fsnotify_conn_mask()
116 	return *fsnotify_conn_mask_p(conn);  in fsnotify_conn_mask()
133 static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,  in fsnotify_update_iref()  argument
136 	bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;  in fsnotify_update_iref()
139 	if (conn->type != FSNOTIFY_OBJ_TYPE_INODE ||  in fsnotify_update_iref()
145 		fsnotify_get_inode_ref(fsnotify_conn_inode(conn));  in fsnotify_update_iref()
146 		conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF;  in fsnotify_update_iref()
149 		inode = fsnotify_conn_inode(conn);  in fsnotify_update_iref()
150 		conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF;  in fsnotify_update_iref()
156 static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)  in __fsnotify_recalc_mask()  argument
162 	assert_spin_locked(&conn->lock);  in __fsnotify_recalc_mask()
164 	if (!fsnotify_valid_obj_type(conn->type))  in __fsnotify_recalc_mask()
166 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in __fsnotify_recalc_mask()
170 		if (conn->type == FSNOTIFY_OBJ_TYPE_INODE &&  in __fsnotify_recalc_mask()
174 	*fsnotify_conn_mask_p(conn) = new_mask;  in __fsnotify_recalc_mask()
176 	return fsnotify_update_iref(conn, want_iref);  in __fsnotify_recalc_mask()
185 void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)  in fsnotify_recalc_mask()  argument
187 	if (!conn)  in fsnotify_recalc_mask()
190 	spin_lock(&conn->lock);  in fsnotify_recalc_mask()
191 	__fsnotify_recalc_mask(conn);  in fsnotify_recalc_mask()
192 	spin_unlock(&conn->lock);  in fsnotify_recalc_mask()
193 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)  in fsnotify_recalc_mask()
195 					fsnotify_conn_inode(conn));  in fsnotify_recalc_mask()
201 	struct fsnotify_mark_connector *conn, *free;  in fsnotify_connector_destroy_workfn()  local
204 	conn = connector_destroy_list;  in fsnotify_connector_destroy_workfn()
209 	while (conn) {  in fsnotify_connector_destroy_workfn()
210 		free = conn;  in fsnotify_connector_destroy_workfn()
211 		conn = conn->destroy_next;  in fsnotify_connector_destroy_workfn()
225 static void fsnotify_get_sb_connectors(struct fsnotify_mark_connector *conn)  in fsnotify_get_sb_connectors()  argument
227 	struct super_block *sb = fsnotify_connector_sb(conn);  in fsnotify_get_sb_connectors()
233 static void fsnotify_put_sb_connectors(struct fsnotify_mark_connector *conn)  in fsnotify_put_sb_connectors()  argument
235 	struct super_block *sb = fsnotify_connector_sb(conn);  in fsnotify_put_sb_connectors()
242 					struct fsnotify_mark_connector *conn,  in fsnotify_detach_connector_from_object()  argument
247 	*type = conn->type;  in fsnotify_detach_connector_from_object()
248 	if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED)  in fsnotify_detach_connector_from_object()
251 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {  in fsnotify_detach_connector_from_object()
252 		inode = fsnotify_conn_inode(conn);  in fsnotify_detach_connector_from_object()
256 		if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF))  in fsnotify_detach_connector_from_object()
258 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {  in fsnotify_detach_connector_from_object()
259 		fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;  in fsnotify_detach_connector_from_object()
260 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {  in fsnotify_detach_connector_from_object()
261 		fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;  in fsnotify_detach_connector_from_object()
264 	fsnotify_put_sb_connectors(conn);  in fsnotify_detach_connector_from_object()
265 	rcu_assign_pointer(*(conn->obj), NULL);  in fsnotify_detach_connector_from_object()
266 	conn->obj = NULL;  in fsnotify_detach_connector_from_object()
267 	conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;  in fsnotify_detach_connector_from_object()
295 	struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);  in fsnotify_put_mark()  local
301 	if (!conn) {  in fsnotify_put_mark()
311 	if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))  in fsnotify_put_mark()
315 	if (hlist_empty(&conn->list)) {  in fsnotify_put_mark()
316 		objp = fsnotify_detach_connector_from_object(conn, &type);  in fsnotify_put_mark()
319 		objp = __fsnotify_recalc_mask(conn);  in fsnotify_put_mark()
320 		type = conn->type;  in fsnotify_put_mark()
323 	spin_unlock(&conn->lock);  in fsnotify_put_mark()
329 		conn->destroy_next = connector_destroy_list;  in fsnotify_put_mark()
330 		connector_destroy_list = conn;  in fsnotify_put_mark()
543 	struct fsnotify_mark_connector *conn;  in fsnotify_attach_connector_to_object()  local
545 	conn = kmem_cache_alloc(fsnotify_mark_connector_cachep, GFP_KERNEL);  in fsnotify_attach_connector_to_object()
546 	if (!conn)  in fsnotify_attach_connector_to_object()
548 	spin_lock_init(&conn->lock);  in fsnotify_attach_connector_to_object()
549 	INIT_HLIST_HEAD(&conn->list);  in fsnotify_attach_connector_to_object()
550 	conn->flags = 0;  in fsnotify_attach_connector_to_object()
551 	conn->type = obj_type;  in fsnotify_attach_connector_to_object()
552 	conn->obj = connp;  in fsnotify_attach_connector_to_object()
555 		conn->fsid = *fsid;  in fsnotify_attach_connector_to_object()
556 		conn->flags = FSNOTIFY_CONN_FLAG_HAS_FSID;  in fsnotify_attach_connector_to_object()
558 		conn->fsid.val[0] = conn->fsid.val[1] = 0;  in fsnotify_attach_connector_to_object()
559 		conn->flags = 0;  in fsnotify_attach_connector_to_object()
561 	fsnotify_get_sb_connectors(conn);  in fsnotify_attach_connector_to_object()
567 	if (cmpxchg(connp, NULL, conn)) {  in fsnotify_attach_connector_to_object()
569 		fsnotify_put_sb_connectors(conn);  in fsnotify_attach_connector_to_object()
570 		kmem_cache_free(fsnotify_mark_connector_cachep, conn);  in fsnotify_attach_connector_to_object()
585 	struct fsnotify_mark_connector *conn;  in fsnotify_grab_connector()  local
589 	conn = srcu_dereference(*connp, &fsnotify_mark_srcu);  in fsnotify_grab_connector()
590 	if (!conn)  in fsnotify_grab_connector()
592 	spin_lock(&conn->lock);  in fsnotify_grab_connector()
593 	if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {  in fsnotify_grab_connector()
594 		spin_unlock(&conn->lock);  in fsnotify_grab_connector()
600 	return conn;  in fsnotify_grab_connector()
615 	struct fsnotify_mark_connector *conn;  in fsnotify_add_mark_list()  local
628 	conn = fsnotify_grab_connector(connp);  in fsnotify_add_mark_list()
629 	if (!conn) {  in fsnotify_add_mark_list()
636 	} else if (fsid && !(conn->flags & FSNOTIFY_CONN_FLAG_HAS_FSID)) {  in fsnotify_add_mark_list()
637 		conn->fsid = *fsid;  in fsnotify_add_mark_list()
640 		conn->flags |= FSNOTIFY_CONN_FLAG_HAS_FSID;  in fsnotify_add_mark_list()
641 	} else if (fsid && (conn->flags & FSNOTIFY_CONN_FLAG_HAS_FSID) &&  in fsnotify_add_mark_list()
642 		   (fsid->val[0] != conn->fsid.val[0] ||  in fsnotify_add_mark_list()
643 		    fsid->val[1] != conn->fsid.val[1])) {  in fsnotify_add_mark_list()
647 		 * Only allow setting conn->fsid once to non zero fsid.  in fsnotify_add_mark_list()
649 		 * conn->fsid.  in fsnotify_add_mark_list()
652 				    "%x.%x != %x.%x\n", __func__, conn->type,  in fsnotify_add_mark_list()
654 				    conn->fsid.val[0], conn->fsid.val[1]);  in fsnotify_add_mark_list()
660 	if (hlist_empty(&conn->list)) {  in fsnotify_add_mark_list()
661 		hlist_add_head_rcu(&mark->obj_list, &conn->list);  in fsnotify_add_mark_list()
666 	hlist_for_each_entry(lmark, &conn->list, obj_list) {  in fsnotify_add_mark_list()
692 	WRITE_ONCE(mark->connector, conn);  in fsnotify_add_mark_list()
694 	spin_unlock(&conn->lock);  in fsnotify_add_mark_list()
765 	struct fsnotify_mark_connector *conn;  in fsnotify_find_mark()  local
768 	conn = fsnotify_grab_connector(connp);  in fsnotify_find_mark()
769 	if (!conn)  in fsnotify_find_mark()
772 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_find_mark()
776 			spin_unlock(&conn->lock);  in fsnotify_find_mark()
780 	spin_unlock(&conn->lock);  in fsnotify_find_mark()
833 	struct fsnotify_mark_connector *conn;  in fsnotify_destroy_marks()  local
838 	conn = fsnotify_grab_connector(connp);  in fsnotify_destroy_marks()
839 	if (!conn)  in fsnotify_destroy_marks()
843 	 * fsnotify_clear_marks_by_group() and once we drop the conn->lock, the  in fsnotify_destroy_marks()
846 	 * iteration after regaining conn->lock.  in fsnotify_destroy_marks()
848 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_destroy_marks()
850 		spin_unlock(&conn->lock);  in fsnotify_destroy_marks()
855 		spin_lock(&conn->lock);  in fsnotify_destroy_marks()
862 	objp = fsnotify_detach_connector_from_object(conn, &type);  in fsnotify_destroy_marks()
863 	spin_unlock(&conn->lock);  in fsnotify_destroy_marks()