34 static bool __kernfs_active(struct kernfs_node *kn)  in __kernfs_active()  argument
36 	return atomic_read(&kn->active) >= 0;  in __kernfs_active()
39 static bool kernfs_active(struct kernfs_node *kn)  in kernfs_active()  argument
41 	lockdep_assert_held(&kernfs_root(kn)->kernfs_rwsem);  in kernfs_active()
42 	return __kernfs_active(kn);  in kernfs_active()
45 static bool kernfs_lockdep(struct kernfs_node *kn)  in kernfs_lockdep()  argument
48 	return kn->flags & KERNFS_LOCKDEP;  in kernfs_lockdep()
54 static int kernfs_name_locked(struct kernfs_node *kn, char *buf, size_t buflen)  in kernfs_name_locked()  argument
56 	if (!kn)  in kernfs_name_locked()
59 	return strlcpy(buf, kn->parent ? kn->name : "/", buflen);  in kernfs_name_locked()
83 	da = kernfs_depth(ra->kn, a);  in kernfs_common_ancestor()
84 	db = kernfs_depth(rb->kn, b);  in kernfs_common_ancestor()
138 	struct kernfs_node *kn, *common;  in kernfs_path_from_node_locked()  local
147 		kn_from = kernfs_root(kn_to)->kn;  in kernfs_path_from_node_locked()
170 		for (kn = kn_to, j = 0; j < i; j++)  in kernfs_path_from_node_locked()
171 			kn = kn->parent;  in kernfs_path_from_node_locked()
174 		len += strlcpy(buf + len, kn->name,  in kernfs_path_from_node_locked()
195 int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen)  in kernfs_name()  argument
201 	ret = kernfs_name_locked(kn, buf, buflen);  in kernfs_name()
241 void pr_cont_kernfs_name(struct kernfs_node *kn)  in pr_cont_kernfs_name()  argument
247 	kernfs_name(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf));  in pr_cont_kernfs_name()
259 void pr_cont_kernfs_path(struct kernfs_node *kn)  in pr_cont_kernfs_path()  argument
266 	sz = kernfs_path_from_node(kn, NULL, kernfs_pr_cont_buf,  in pr_cont_kernfs_path()
291 struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn)  in kernfs_get_parent()  argument
297 	parent = kn->parent;  in kernfs_get_parent()
328 			       const void *ns, const struct kernfs_node *kn)  in kernfs_name_compare()  argument
330 	if (hash < kn->hash)  in kernfs_name_compare()
332 	if (hash > kn->hash)  in kernfs_name_compare()
334 	if (ns < kn->ns)  in kernfs_name_compare()
336 	if (ns > kn->ns)  in kernfs_name_compare()
338 	return strcmp(name, kn->name);  in kernfs_name_compare()
360 static int kernfs_link_sibling(struct kernfs_node *kn)  in kernfs_link_sibling()  argument
362 	struct rb_node **node = &kn->parent->dir.children.rb_node;  in kernfs_link_sibling()
371 		result = kernfs_sd_compare(kn, pos);  in kernfs_link_sibling()
381 	rb_link_node(&kn->rb, parent, node);  in kernfs_link_sibling()
382 	rb_insert_color(&kn->rb, &kn->parent->dir.children);  in kernfs_link_sibling()
385 	if (kernfs_type(kn) == KERNFS_DIR)  in kernfs_link_sibling()
386 		kn->parent->dir.subdirs++;  in kernfs_link_sibling()
387 	kernfs_inc_rev(kn->parent);  in kernfs_link_sibling()
403 static bool kernfs_unlink_sibling(struct kernfs_node *kn)  in kernfs_unlink_sibling()  argument
405 	if (RB_EMPTY_NODE(&kn->rb))  in kernfs_unlink_sibling()
408 	if (kernfs_type(kn) == KERNFS_DIR)  in kernfs_unlink_sibling()
409 		kn->parent->dir.subdirs--;  in kernfs_unlink_sibling()
410 	kernfs_inc_rev(kn->parent);  in kernfs_unlink_sibling()
412 	rb_erase(&kn->rb, &kn->parent->dir.children);  in kernfs_unlink_sibling()
413 	RB_CLEAR_NODE(&kn->rb);  in kernfs_unlink_sibling()
427 struct kernfs_node *kernfs_get_active(struct kernfs_node *kn)  in kernfs_get_active()  argument
429 	if (unlikely(!kn))  in kernfs_get_active()
432 	if (!atomic_inc_unless_negative(&kn->active))  in kernfs_get_active()
435 	if (kernfs_lockdep(kn))  in kernfs_get_active()
436 		rwsem_acquire_read(&kn->dep_map, 0, 1, _RET_IP_);  in kernfs_get_active()
437 	return kn;  in kernfs_get_active()
447 void kernfs_put_active(struct kernfs_node *kn)  in kernfs_put_active()  argument
451 	if (unlikely(!kn))  in kernfs_put_active()
454 	if (kernfs_lockdep(kn))  in kernfs_put_active()
455 		rwsem_release(&kn->dep_map, _RET_IP_);  in kernfs_put_active()
456 	v = atomic_dec_return(&kn->active);  in kernfs_put_active()
460 	wake_up_all(&kernfs_root(kn)->deactivate_waitq);  in kernfs_put_active()
471 static void kernfs_drain(struct kernfs_node *kn)  in kernfs_drain()  argument
472 	__releases(&kernfs_root(kn)->kernfs_rwsem)  in kernfs_drain()
473 	__acquires(&kernfs_root(kn)->kernfs_rwsem)  in kernfs_drain()
475 	struct kernfs_root *root = kernfs_root(kn);  in kernfs_drain()
478 	WARN_ON_ONCE(kernfs_active(kn));  in kernfs_drain()
486 	if (atomic_read(&kn->active) == KN_DEACTIVATED_BIAS &&  in kernfs_drain()
487 	    !kernfs_should_drain_open_files(kn))  in kernfs_drain()
492 	if (kernfs_lockdep(kn)) {  in kernfs_drain()
493 		rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_);  in kernfs_drain()
494 		if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS)  in kernfs_drain()
495 			lock_contended(&kn->dep_map, _RET_IP_);  in kernfs_drain()
499 		   atomic_read(&kn->active) == KN_DEACTIVATED_BIAS);  in kernfs_drain()
501 	if (kernfs_lockdep(kn)) {  in kernfs_drain()
502 		lock_acquired(&kn->dep_map, _RET_IP_);  in kernfs_drain()
503 		rwsem_release(&kn->dep_map, _RET_IP_);  in kernfs_drain()
506 	if (kernfs_should_drain_open_files(kn))  in kernfs_drain()
507 		kernfs_drain_open_files(kn);  in kernfs_drain()
516 void kernfs_get(struct kernfs_node *kn)  in kernfs_get()  argument
518 	if (kn) {  in kernfs_get()
519 		WARN_ON(!atomic_read(&kn->count));  in kernfs_get()
520 		atomic_inc(&kn->count);  in kernfs_get()
531 void kernfs_put(struct kernfs_node *kn)  in kernfs_put()  argument
536 	if (!kn || !atomic_dec_and_test(&kn->count))  in kernfs_put()
538 	root = kernfs_root(kn);  in kernfs_put()
544 	parent = kn->parent;  in kernfs_put()
546 	WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS,  in kernfs_put()
548 		  parent ? parent->name : "", kn->name, atomic_read(&kn->active));  in kernfs_put()
550 	if (kernfs_type(kn) == KERNFS_LINK)  in kernfs_put()
551 		kernfs_put(kn->symlink.target_kn);  in kernfs_put()
553 	kfree_const(kn->name);  in kernfs_put()
555 	if (kn->iattr) {  in kernfs_put()
556 		simple_xattrs_free(&kn->iattr->xattrs);  in kernfs_put()
557 		kmem_cache_free(kernfs_iattrs_cache, kn->iattr);  in kernfs_put()
560 	idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));  in kernfs_put()
562 	kmem_cache_free(kernfs_node_cache, kn);  in kernfs_put()
564 	kn = parent;  in kernfs_put()
565 	if (kn) {  in kernfs_put()
566 		if (atomic_dec_and_test(&kn->count))  in kernfs_put()
600 	struct kernfs_node *kn;  in __kernfs_new_node()  local
608 	kn = kmem_cache_zalloc(kernfs_node_cache, GFP_KERNEL);  in __kernfs_new_node()
609 	if (!kn)  in __kernfs_new_node()
614 	ret = idr_alloc_cyclic(&root->ino_idr, kn, 1, 0, GFP_ATOMIC);  in __kernfs_new_node()
624 	kn->id = (u64)id_highbits << 32 | ret;  in __kernfs_new_node()
626 	atomic_set(&kn->count, 1);  in __kernfs_new_node()
627 	atomic_set(&kn->active, KN_DEACTIVATED_BIAS);  in __kernfs_new_node()
628 	RB_CLEAR_NODE(&kn->rb);  in __kernfs_new_node()
630 	kn->name = name;  in __kernfs_new_node()
631 	kn->mode = mode;  in __kernfs_new_node()
632 	kn->flags = flags;  in __kernfs_new_node()
641 		ret = __kernfs_setattr(kn, &iattr);  in __kernfs_new_node()
647 		ret = security_kernfs_init_security(parent, kn);  in __kernfs_new_node()
652 	return kn;  in __kernfs_new_node()
655 	idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));  in __kernfs_new_node()
657 	kmem_cache_free(kernfs_node_cache, kn);  in __kernfs_new_node()
668 	struct kernfs_node *kn;  in kernfs_new_node()  local
670 	kn = __kernfs_new_node(kernfs_root(parent), parent,  in kernfs_new_node()
672 	if (kn) {  in kernfs_new_node()
674 		kn->parent = parent;  in kernfs_new_node()
676 	return kn;  in kernfs_new_node()
693 	struct kernfs_node *kn;  in kernfs_find_and_get_node_by_id()  local
699 	kn = idr_find(&root->ino_idr, (u32)ino);  in kernfs_find_and_get_node_by_id()
700 	if (!kn)  in kernfs_find_and_get_node_by_id()
705 		if (kernfs_ino(kn) != ino)  in kernfs_find_and_get_node_by_id()
709 		if (unlikely(gen && kernfs_gen(kn) != gen))  in kernfs_find_and_get_node_by_id()
718 	if (unlikely(!__kernfs_active(kn) || !atomic_inc_not_zero(&kn->count)))  in kernfs_find_and_get_node_by_id()
722 	return kn;  in kernfs_find_and_get_node_by_id()
740 int kernfs_add_one(struct kernfs_node *kn)  in kernfs_add_one()  argument
742 	struct kernfs_node *parent = kn->parent;  in kernfs_add_one()
752 	if (WARN(has_ns != (bool)kn->ns, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",  in kernfs_add_one()
753 		 has_ns ? "required" : "invalid", parent->name, kn->name))  in kernfs_add_one()
763 	kn->hash = kernfs_name_hash(kn->name, kn->ns);  in kernfs_add_one()
765 	ret = kernfs_link_sibling(kn);  in kernfs_add_one()
785 	if (!(kernfs_root(kn)->flags & KERNFS_ROOT_CREATE_DEACTIVATED))  in kernfs_add_one()
786 		kernfs_activate(kn);  in kernfs_add_one()
821 		struct kernfs_node *kn;  in kernfs_find_ns()  local
824 		kn = rb_to_kn(node);  in kernfs_find_ns()
825 		result = kernfs_name_compare(hash, name, ns, kn);  in kernfs_find_ns()
831 			return kn;  in kernfs_find_ns()
880 	struct kernfs_node *kn;  in kernfs_find_and_get_ns()  local
884 	kn = kernfs_find_ns(parent, name, ns);  in kernfs_find_and_get_ns()
885 	kernfs_get(kn);  in kernfs_find_and_get_ns()
888 	return kn;  in kernfs_find_and_get_ns()
905 	struct kernfs_node *kn;  in kernfs_walk_and_get_ns()  local
909 	kn = kernfs_walk_ns(parent, path, ns);  in kernfs_walk_and_get_ns()
910 	kernfs_get(kn);  in kernfs_walk_and_get_ns()
913 	return kn;  in kernfs_walk_and_get_ns()
929 	struct kernfs_node *kn;  in kernfs_create_root()  local
950 	kn = __kernfs_new_node(root, NULL, "", S_IFDIR | S_IRUGO | S_IXUGO,  in kernfs_create_root()
953 	if (!kn) {  in kernfs_create_root()
959 	kn->priv = priv;  in kernfs_create_root()
960 	kn->dir.root = root;  in kernfs_create_root()
964 	root->kn = kn;  in kernfs_create_root()
968 		kernfs_activate(kn);  in kernfs_create_root()
986 	kernfs_get(root->kn);  in kernfs_destroy_root()
987 	kernfs_remove(root->kn);  in kernfs_destroy_root()
988 	kernfs_put(root->kn); /* will also free @root */  in kernfs_destroy_root()
997 	return root->kn;  in kernfs_root_to_node()
1017 	struct kernfs_node *kn;  in kernfs_create_dir_ns()  local
1021 	kn = kernfs_new_node(parent, name, mode | S_IFDIR,  in kernfs_create_dir_ns()
1023 	if (!kn)  in kernfs_create_dir_ns()
1026 	kn->dir.root = parent->dir.root;  in kernfs_create_dir_ns()
1027 	kn->ns = ns;  in kernfs_create_dir_ns()
1028 	kn->priv = priv;  in kernfs_create_dir_ns()
1031 	rc = kernfs_add_one(kn);  in kernfs_create_dir_ns()
1033 		return kn;  in kernfs_create_dir_ns()
1035 	kernfs_put(kn);  in kernfs_create_dir_ns()
1049 	struct kernfs_node *kn;  in kernfs_create_empty_dir()  local
1053 	kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR,  in kernfs_create_empty_dir()
1055 	if (!kn)  in kernfs_create_empty_dir()
1058 	kn->flags |= KERNFS_EMPTY_DIR;  in kernfs_create_empty_dir()
1059 	kn->dir.root = parent->dir.root;  in kernfs_create_empty_dir()
1060 	kn->ns = NULL;  in kernfs_create_empty_dir()
1061 	kn->priv = NULL;  in kernfs_create_empty_dir()
1064 	rc = kernfs_add_one(kn);  in kernfs_create_empty_dir()
1066 		return kn;  in kernfs_create_empty_dir()
1068 	kernfs_put(kn);  in kernfs_create_empty_dir()
1074 	struct kernfs_node *kn;  in kernfs_dop_revalidate()  local
1107 	kn = kernfs_dentry_node(dentry);  in kernfs_dop_revalidate()
1108 	root = kernfs_root(kn);  in kernfs_dop_revalidate()
1112 	if (!kernfs_active(kn))  in kernfs_dop_revalidate()
1116 	if (kernfs_dentry_node(dentry->d_parent) != kn->parent)  in kernfs_dop_revalidate()
1120 	if (strcmp(dentry->d_name.name, kn->name) != 0)  in kernfs_dop_revalidate()
1124 	if (kn->parent && kernfs_ns_enabled(kn->parent) &&  in kernfs_dop_revalidate()
1125 	    kernfs_info(dentry->d_sb)->ns != kn->ns)  in kernfs_dop_revalidate()
1144 	struct kernfs_node *kn;  in kernfs_iop_lookup()  local
1154 	kn = kernfs_find_ns(parent, dentry->d_name.name, ns);  in kernfs_iop_lookup()
1156 	if (kn) {  in kernfs_iop_lookup()
1160 		if (!kernfs_active(kn)) {  in kernfs_iop_lookup()
1164 		inode = kernfs_get_inode(dir->i_sb, kn);  in kernfs_iop_lookup()
1204 	struct kernfs_node *kn  = kernfs_dentry_node(dentry);  in kernfs_iop_rmdir()  local
1205 	struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;  in kernfs_iop_rmdir()
1211 	if (!kernfs_get_active(kn))  in kernfs_iop_rmdir()
1214 	ret = scops->rmdir(kn);  in kernfs_iop_rmdir()
1216 	kernfs_put_active(kn);  in kernfs_iop_rmdir()
1225 	struct kernfs_node *kn = kernfs_dentry_node(old_dentry);  in kernfs_iop_rename()  local
1227 	struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;  in kernfs_iop_rename()
1236 	if (!kernfs_get_active(kn))  in kernfs_iop_rename()
1240 		kernfs_put_active(kn);  in kernfs_iop_rename()
1244 	ret = scops->rename(kn, new_parent, new_dentry->d_name.name);  in kernfs_iop_rename()
1247 	kernfs_put_active(kn);  in kernfs_iop_rename()
1318 static void kernfs_activate_one(struct kernfs_node *kn)  in kernfs_activate_one()  argument
1320 	lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem);  in kernfs_activate_one()
1322 	kn->flags |= KERNFS_ACTIVATED;  in kernfs_activate_one()
1324 	if (kernfs_active(kn) || (kn->flags & (KERNFS_HIDDEN | KERNFS_REMOVING)))  in kernfs_activate_one()
1327 	WARN_ON_ONCE(kn->parent && RB_EMPTY_NODE(&kn->rb));  in kernfs_activate_one()
1328 	WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS);  in kernfs_activate_one()
1330 	atomic_sub(KN_DEACTIVATED_BIAS, &kn->active);  in kernfs_activate_one()
1346 void kernfs_activate(struct kernfs_node *kn)  in kernfs_activate()  argument
1349 	struct kernfs_root *root = kernfs_root(kn);  in kernfs_activate()
1354 	while ((pos = kernfs_next_descendant_post(pos, kn)))  in kernfs_activate()
1372 void kernfs_show(struct kernfs_node *kn, bool show)  in kernfs_show()  argument
1374 	struct kernfs_root *root = kernfs_root(kn);  in kernfs_show()
1376 	if (WARN_ON_ONCE(kernfs_type(kn) == KERNFS_DIR))  in kernfs_show()
1382 		kn->flags &= ~KERNFS_HIDDEN;  in kernfs_show()
1383 		if (kn->flags & KERNFS_ACTIVATED)  in kernfs_show()
1384 			kernfs_activate_one(kn);  in kernfs_show()
1386 		kn->flags |= KERNFS_HIDDEN;  in kernfs_show()
1387 		if (kernfs_active(kn))  in kernfs_show()
1388 			atomic_add(KN_DEACTIVATED_BIAS, &kn->active);  in kernfs_show()
1389 		kernfs_drain(kn);  in kernfs_show()
1395 static void __kernfs_remove(struct kernfs_node *kn)  in __kernfs_remove()  argument
1400 	if (!kn)  in __kernfs_remove()
1403 	lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem);  in __kernfs_remove()
1409 	if (kn->parent && RB_EMPTY_NODE(&kn->rb))  in __kernfs_remove()
1412 	pr_debug("kernfs %s: removing\n", kn->name);  in __kernfs_remove()
1416 	while ((pos = kernfs_next_descendant_post(pos, kn))) {  in __kernfs_remove()
1424 		pos = kernfs_leftmost_descendant(kn);  in __kernfs_remove()
1454 	} while (pos != kn);  in __kernfs_remove()
1463 void kernfs_remove(struct kernfs_node *kn)  in kernfs_remove()  argument
1467 	if (!kn)  in kernfs_remove()
1470 	root = kernfs_root(kn);  in kernfs_remove()
1473 	__kernfs_remove(kn);  in kernfs_remove()
1491 void kernfs_break_active_protection(struct kernfs_node *kn)  in kernfs_break_active_protection()  argument
1497 	kernfs_put_active(kn);  in kernfs_break_active_protection()
1515 void kernfs_unbreak_active_protection(struct kernfs_node *kn)  in kernfs_unbreak_active_protection()  argument
1526 	atomic_inc(&kn->active);  in kernfs_unbreak_active_protection()
1527 	if (kernfs_lockdep(kn))  in kernfs_unbreak_active_protection()
1528 		rwsem_acquire(&kn->dep_map, 0, 1, _RET_IP_);  in kernfs_unbreak_active_protection()
1557 bool kernfs_remove_self(struct kernfs_node *kn)  in kernfs_remove_self()  argument
1560 	struct kernfs_root *root = kernfs_root(kn);  in kernfs_remove_self()
1563 	kernfs_break_active_protection(kn);  in kernfs_remove_self()
1574 	if (!(kn->flags & KERNFS_SUICIDAL)) {  in kernfs_remove_self()
1575 		kn->flags |= KERNFS_SUICIDAL;  in kernfs_remove_self()
1576 		__kernfs_remove(kn);  in kernfs_remove_self()
1577 		kn->flags |= KERNFS_SUICIDED;  in kernfs_remove_self()
1580 		wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq;  in kernfs_remove_self()
1586 			if ((kn->flags & KERNFS_SUICIDED) &&  in kernfs_remove_self()
1587 			    atomic_read(&kn->active) == KN_DEACTIVATED_BIAS)  in kernfs_remove_self()
1595 		WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb));  in kernfs_remove_self()
1603 	kernfs_unbreak_active_protection(kn);  in kernfs_remove_self()
1621 	struct kernfs_node *kn;  in kernfs_remove_by_name_ns()  local
1633 	kn = kernfs_find_ns(parent, name, ns);  in kernfs_remove_by_name_ns()
1634 	if (kn) {  in kernfs_remove_by_name_ns()
1635 		kernfs_get(kn);  in kernfs_remove_by_name_ns()
1636 		__kernfs_remove(kn);  in kernfs_remove_by_name_ns()
1637 		kernfs_put(kn);  in kernfs_remove_by_name_ns()
1642 	if (kn)  in kernfs_remove_by_name_ns()
1655 int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,  in kernfs_rename_ns()  argument
1664 	if (!kn->parent)  in kernfs_rename_ns()
1667 	root = kernfs_root(kn);  in kernfs_rename_ns()
1671 	if (!kernfs_active(kn) || !kernfs_active(new_parent) ||  in kernfs_rename_ns()
1676 	if ((kn->parent == new_parent) && (kn->ns == new_ns) &&  in kernfs_rename_ns()
1677 	    (strcmp(kn->name, new_name) == 0))  in kernfs_rename_ns()
1685 	if (strcmp(kn->name, new_name) != 0) {  in kernfs_rename_ns()
1697 	kernfs_unlink_sibling(kn);  in kernfs_rename_ns()
1703 	old_parent = kn->parent;  in kernfs_rename_ns()
1704 	kn->parent = new_parent;  in kernfs_rename_ns()
1706 	kn->ns = new_ns;  in kernfs_rename_ns()
1708 		old_name = kn->name;  in kernfs_rename_ns()
1709 		kn->name = new_name;  in kernfs_rename_ns()
1714 	kn->hash = kernfs_name_hash(kn->name, kn->ns);  in kernfs_rename_ns()
1715 	kernfs_link_sibling(kn);  in kernfs_rename_ns()
1727 static inline unsigned char dt_type(struct kernfs_node *kn)  in dt_type()  argument
1729 	return (kn->mode >> 12) & 15;  in dt_type()