143 static int no_open(struct inode *inode, struct file *file)  in no_open()  argument
156 int inode_init_always(struct super_block *sb, struct inode *inode)  in inode_init_always()  argument
160 	struct address_space *const mapping = &inode->i_data;  in inode_init_always()
162 	inode->i_sb = sb;  in inode_init_always()
163 	inode->i_blkbits = sb->s_blocksize_bits;  in inode_init_always()
164 	inode->i_flags = 0;  in inode_init_always()
165 	atomic64_set(&inode->i_sequence, 0);  in inode_init_always()
166 	atomic_set(&inode->i_count, 1);  in inode_init_always()
167 	inode->i_op = &empty_iops;  in inode_init_always()
168 	inode->i_fop = &no_open_fops;  in inode_init_always()
169 	inode->i_ino = 0;  in inode_init_always()
170 	inode->__i_nlink = 1;  in inode_init_always()
171 	inode->i_opflags = 0;  in inode_init_always()
173 		inode->i_opflags |= IOP_XATTR;  in inode_init_always()
174 	i_uid_write(inode, 0);  in inode_init_always()
175 	i_gid_write(inode, 0);  in inode_init_always()
176 	atomic_set(&inode->i_writecount, 0);  in inode_init_always()
177 	inode->i_size = 0;  in inode_init_always()
178 	inode->i_write_hint = WRITE_LIFE_NOT_SET;  in inode_init_always()
179 	inode->i_blocks = 0;  in inode_init_always()
180 	inode->i_bytes = 0;  in inode_init_always()
181 	inode->i_generation = 0;  in inode_init_always()
182 	inode->i_pipe = NULL;  in inode_init_always()
183 	inode->i_cdev = NULL;  in inode_init_always()
184 	inode->i_link = NULL;  in inode_init_always()
185 	inode->i_dir_seq = 0;  in inode_init_always()
186 	inode->i_rdev = 0;  in inode_init_always()
187 	inode->dirtied_when = 0;  in inode_init_always()
190 	inode->i_wb_frn_winner = 0;  in inode_init_always()
191 	inode->i_wb_frn_avg_time = 0;  in inode_init_always()
192 	inode->i_wb_frn_history = 0;  in inode_init_always()
195 	spin_lock_init(&inode->i_lock);  in inode_init_always()
196 	lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);  in inode_init_always()
198 	init_rwsem(&inode->i_rwsem);  in inode_init_always()
199 	lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);  in inode_init_always()
201 	atomic_set(&inode->i_dio_count, 0);  in inode_init_always()
204 	mapping->host = inode;  in inode_init_always()
218 	inode->i_private = NULL;  in inode_init_always()
219 	inode->i_mapping = mapping;  in inode_init_always()
220 	INIT_HLIST_HEAD(&inode->i_dentry);	/* buggered by rcu freeing */  in inode_init_always()
222 	inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;  in inode_init_always()
226 	inode->i_fsnotify_mask = 0;  in inode_init_always()
228 	inode->i_flctx = NULL;  in inode_init_always()
230 	if (unlikely(security_inode_alloc(inode)))  in inode_init_always()
238 void free_inode_nonrcu(struct inode *inode)  in free_inode_nonrcu()  argument
240 	kmem_cache_free(inode_cachep, inode);  in free_inode_nonrcu()
246 	struct inode *inode = container_of(head, struct inode, i_rcu);  in i_callback()  local
247 	if (inode->free_inode)  in i_callback()
248 		inode->free_inode(inode);  in i_callback()
250 		free_inode_nonrcu(inode);  in i_callback()
253 static struct inode *alloc_inode(struct super_block *sb)  in alloc_inode()
256 	struct inode *inode;  in alloc_inode()  local
259 		inode = ops->alloc_inode(sb);  in alloc_inode()
261 		inode = alloc_inode_sb(sb, inode_cachep, GFP_KERNEL);  in alloc_inode()
263 	if (!inode)  in alloc_inode()
266 	if (unlikely(inode_init_always(sb, inode))) {  in alloc_inode()
268 			ops->destroy_inode(inode);  in alloc_inode()
272 		inode->free_inode = ops->free_inode;  in alloc_inode()
273 		i_callback(&inode->i_rcu);  in alloc_inode()
277 	return inode;  in alloc_inode()
280 void __destroy_inode(struct inode *inode)  in __destroy_inode()  argument
282 	BUG_ON(inode_has_buffers(inode));  in __destroy_inode()
283 	inode_detach_wb(inode);  in __destroy_inode()
284 	security_inode_free(inode);  in __destroy_inode()
285 	fsnotify_inode_delete(inode);  in __destroy_inode()
286 	locks_free_lock_context(inode);  in __destroy_inode()
287 	if (!inode->i_nlink) {  in __destroy_inode()
288 		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);  in __destroy_inode()
289 		atomic_long_dec(&inode->i_sb->s_remove_count);  in __destroy_inode()
293 	if (inode->i_acl && !is_uncached_acl(inode->i_acl))  in __destroy_inode()
294 		posix_acl_release(inode->i_acl);  in __destroy_inode()
295 	if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))  in __destroy_inode()
296 		posix_acl_release(inode->i_default_acl);  in __destroy_inode()
302 static void destroy_inode(struct inode *inode)  in destroy_inode()  argument
304 	const struct super_operations *ops = inode->i_sb->s_op;  in destroy_inode()
306 	BUG_ON(!list_empty(&inode->i_lru));  in destroy_inode()
307 	__destroy_inode(inode);  in destroy_inode()
309 		ops->destroy_inode(inode);  in destroy_inode()
313 	inode->free_inode = ops->free_inode;  in destroy_inode()
314 	call_rcu(&inode->i_rcu, i_callback);  in destroy_inode()
328 void drop_nlink(struct inode *inode)  in drop_nlink()  argument
330 	WARN_ON(inode->i_nlink == 0);  in drop_nlink()
331 	inode->__i_nlink--;  in drop_nlink()
332 	if (!inode->i_nlink)  in drop_nlink()
333 		atomic_long_inc(&inode->i_sb->s_remove_count);  in drop_nlink()
345 void clear_nlink(struct inode *inode)  in clear_nlink()  argument
347 	if (inode->i_nlink) {  in clear_nlink()
348 		inode->__i_nlink = 0;  in clear_nlink()
349 		atomic_long_inc(&inode->i_sb->s_remove_count);  in clear_nlink()
362 void set_nlink(struct inode *inode, unsigned int nlink)  in set_nlink()  argument
365 		clear_nlink(inode);  in set_nlink()
368 		if (inode->i_nlink == 0)  in set_nlink()
369 			atomic_long_dec(&inode->i_sb->s_remove_count);  in set_nlink()
371 		inode->__i_nlink = nlink;  in set_nlink()
384 void inc_nlink(struct inode *inode)  in inc_nlink()  argument
386 	if (unlikely(inode->i_nlink == 0)) {  in inc_nlink()
387 		WARN_ON(!(inode->i_state & I_LINKABLE));  in inc_nlink()
388 		atomic_long_dec(&inode->i_sb->s_remove_count);  in inc_nlink()
391 	inode->__i_nlink++;  in inc_nlink()
416 void inode_init_once(struct inode *inode)  in inode_init_once()  argument
418 	memset(inode, 0, sizeof(*inode));  in inode_init_once()
419 	INIT_HLIST_NODE(&inode->i_hash);  in inode_init_once()
420 	INIT_LIST_HEAD(&inode->i_devices);  in inode_init_once()
421 	INIT_LIST_HEAD(&inode->i_io_list);  in inode_init_once()
422 	INIT_LIST_HEAD(&inode->i_wb_list);  in inode_init_once()
423 	INIT_LIST_HEAD(&inode->i_lru);  in inode_init_once()
424 	INIT_LIST_HEAD(&inode->i_sb_list);  in inode_init_once()
425 	__address_space_init_once(&inode->i_data);  in inode_init_once()
426 	i_size_ordered_init(inode);  in inode_init_once()
432 	struct inode *inode = (struct inode *) foo;  in init_once()  local
434 	inode_init_once(inode);  in init_once()
440 void __iget(struct inode *inode)  in __iget()  argument
442 	atomic_inc(&inode->i_count);  in __iget()
448 void ihold(struct inode *inode)  in ihold()  argument
450 	WARN_ON(atomic_inc_return(&inode->i_count) < 2);  in ihold()
454 static void __inode_add_lru(struct inode *inode, bool rotate)  in __inode_add_lru()  argument
456 	if (inode->i_state & (I_DIRTY_ALL | I_SYNC | I_FREEING | I_WILL_FREE))  in __inode_add_lru()
458 	if (atomic_read(&inode->i_count))  in __inode_add_lru()
460 	if (!(inode->i_sb->s_flags & SB_ACTIVE))  in __inode_add_lru()
462 	if (!mapping_shrinkable(&inode->i_data))  in __inode_add_lru()
465 	if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))  in __inode_add_lru()
468 		inode->i_state |= I_REFERENCED;  in __inode_add_lru()
476 void inode_add_lru(struct inode *inode)  in inode_add_lru()  argument
478 	__inode_add_lru(inode, false);  in inode_add_lru()
481 static void inode_lru_list_del(struct inode *inode)  in inode_lru_list_del()  argument
483 	if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))  in inode_lru_list_del()
491 void inode_sb_list_add(struct inode *inode)  in inode_sb_list_add()  argument
493 	spin_lock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_add()
494 	list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);  in inode_sb_list_add()
495 	spin_unlock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_add()
499 static inline void inode_sb_list_del(struct inode *inode)  in inode_sb_list_del()  argument
501 	if (!list_empty(&inode->i_sb_list)) {  in inode_sb_list_del()
502 		spin_lock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_del()
503 		list_del_init(&inode->i_sb_list);  in inode_sb_list_del()
504 		spin_unlock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_del()
526 void __insert_inode_hash(struct inode *inode, unsigned long hashval)  in __insert_inode_hash()  argument
528 	struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);  in __insert_inode_hash()
531 	spin_lock(&inode->i_lock);  in __insert_inode_hash()
532 	hlist_add_head_rcu(&inode->i_hash, b);  in __insert_inode_hash()
533 	spin_unlock(&inode->i_lock);  in __insert_inode_hash()
544 void __remove_inode_hash(struct inode *inode)  in __remove_inode_hash()  argument
547 	spin_lock(&inode->i_lock);  in __remove_inode_hash()
548 	hlist_del_init_rcu(&inode->i_hash);  in __remove_inode_hash()
549 	spin_unlock(&inode->i_lock);  in __remove_inode_hash()
556 	struct inode *host;  in dump_mapping()
603 void clear_inode(struct inode *inode)  in clear_inode()  argument
610 	xa_lock_irq(&inode->i_data.i_pages);  in clear_inode()
611 	BUG_ON(inode->i_data.nrpages);  in clear_inode()
620 	xa_unlock_irq(&inode->i_data.i_pages);  in clear_inode()
621 	BUG_ON(!list_empty(&inode->i_data.private_list));  in clear_inode()
622 	BUG_ON(!(inode->i_state & I_FREEING));  in clear_inode()
623 	BUG_ON(inode->i_state & I_CLEAR);  in clear_inode()
624 	BUG_ON(!list_empty(&inode->i_wb_list));  in clear_inode()
626 	inode->i_state = I_FREEING | I_CLEAR;  in clear_inode()
643 static void evict(struct inode *inode)  in evict()  argument
645 	const struct super_operations *op = inode->i_sb->s_op;  in evict()
647 	BUG_ON(!(inode->i_state & I_FREEING));  in evict()
648 	BUG_ON(!list_empty(&inode->i_lru));  in evict()
650 	if (!list_empty(&inode->i_io_list))  in evict()
651 		inode_io_list_del(inode);  in evict()
653 	inode_sb_list_del(inode);  in evict()
661 	inode_wait_for_writeback(inode);  in evict()
664 		op->evict_inode(inode);  in evict()
666 		truncate_inode_pages_final(&inode->i_data);  in evict()
667 		clear_inode(inode);  in evict()
669 	if (S_ISCHR(inode->i_mode) && inode->i_cdev)  in evict()
670 		cd_forget(inode);  in evict()
672 	remove_inode_hash(inode);  in evict()
674 	spin_lock(&inode->i_lock);  in evict()
675 	wake_up_bit(&inode->i_state, __I_NEW);  in evict()
676 	BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));  in evict()
677 	spin_unlock(&inode->i_lock);  in evict()
679 	destroy_inode(inode);  in evict()
692 		struct inode *inode;  in dispose_list()  local
694 		inode = list_first_entry(head, struct inode, i_lru);  in dispose_list()
695 		list_del_init(&inode->i_lru);  in dispose_list()
697 		evict(inode);  in dispose_list()
713 	struct inode *inode, *next;  in evict_inodes()  local
718 	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {  in evict_inodes()
719 		if (atomic_read(&inode->i_count))  in evict_inodes()
722 		spin_lock(&inode->i_lock);  in evict_inodes()
723 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {  in evict_inodes()
724 			spin_unlock(&inode->i_lock);  in evict_inodes()
728 		inode->i_state |= I_FREEING;  in evict_inodes()
729 		inode_lru_list_del(inode);  in evict_inodes()
730 		spin_unlock(&inode->i_lock);  in evict_inodes()
731 		list_add(&inode->i_lru, &dispose);  in evict_inodes()
764 	struct inode *inode, *next;  in invalidate_inodes()  local
769 	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {  in invalidate_inodes()
770 		spin_lock(&inode->i_lock);  in invalidate_inodes()
771 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {  in invalidate_inodes()
772 			spin_unlock(&inode->i_lock);  in invalidate_inodes()
775 		if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {  in invalidate_inodes()
776 			spin_unlock(&inode->i_lock);  in invalidate_inodes()
780 		if (atomic_read(&inode->i_count)) {  in invalidate_inodes()
781 			spin_unlock(&inode->i_lock);  in invalidate_inodes()
786 		inode->i_state |= I_FREEING;  in invalidate_inodes()
787 		inode_lru_list_del(inode);  in invalidate_inodes()
788 		spin_unlock(&inode->i_lock);  in invalidate_inodes()
789 		list_add(&inode->i_lru, &dispose);  in invalidate_inodes()
819 	struct inode	*inode = container_of(item, struct inode, i_lru);  in inode_lru_isolate()  local
825 	if (!spin_trylock(&inode->i_lock))  in inode_lru_isolate()
834 	if (atomic_read(&inode->i_count) ||  in inode_lru_isolate()
835 	    (inode->i_state & ~I_REFERENCED) ||  in inode_lru_isolate()
836 	    !mapping_shrinkable(&inode->i_data)) {  in inode_lru_isolate()
837 		list_lru_isolate(lru, &inode->i_lru);  in inode_lru_isolate()
838 		spin_unlock(&inode->i_lock);  in inode_lru_isolate()
844 	if (inode->i_state & I_REFERENCED) {  in inode_lru_isolate()
845 		inode->i_state &= ~I_REFERENCED;  in inode_lru_isolate()
846 		spin_unlock(&inode->i_lock);  in inode_lru_isolate()
855 	if (inode_has_buffers(inode) || !mapping_empty(&inode->i_data)) {  in inode_lru_isolate()
856 		__iget(inode);  in inode_lru_isolate()
857 		spin_unlock(&inode->i_lock);  in inode_lru_isolate()
859 		if (remove_inode_buffers(inode)) {  in inode_lru_isolate()
861 			reap = invalidate_mapping_pages(&inode->i_data, 0, -1);  in inode_lru_isolate()
869 		iput(inode);  in inode_lru_isolate()
874 	WARN_ON(inode->i_state & I_NEW);  in inode_lru_isolate()
875 	inode->i_state |= I_FREEING;  in inode_lru_isolate()
876 	list_lru_isolate_move(lru, &inode->i_lru, freeable);  in inode_lru_isolate()
877 	spin_unlock(&inode->i_lock);  in inode_lru_isolate()
900 static void __wait_on_freeing_inode(struct inode *inode);
904 static struct inode *find_inode(struct super_block *sb,  in find_inode()
906 				int (*test)(struct inode *, void *),  in find_inode()  argument
909 	struct inode *inode = NULL;  in find_inode()  local
912 	hlist_for_each_entry(inode, head, i_hash) {  in find_inode()
913 		if (inode->i_sb != sb)  in find_inode()
915 		if (!test(inode, data))  in find_inode()
917 		spin_lock(&inode->i_lock);  in find_inode()
918 		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {  in find_inode()
919 			__wait_on_freeing_inode(inode);  in find_inode()
922 		if (unlikely(inode->i_state & I_CREATING)) {  in find_inode()
923 			spin_unlock(&inode->i_lock);  in find_inode()
926 		__iget(inode);  in find_inode()
927 		spin_unlock(&inode->i_lock);  in find_inode()
928 		return inode;  in find_inode()
937 static struct inode *find_inode_fast(struct super_block *sb,  in find_inode_fast()
940 	struct inode *inode = NULL;  in find_inode_fast()  local
943 	hlist_for_each_entry(inode, head, i_hash) {  in find_inode_fast()
944 		if (inode->i_ino != ino)  in find_inode_fast()
946 		if (inode->i_sb != sb)  in find_inode_fast()
948 		spin_lock(&inode->i_lock);  in find_inode_fast()
949 		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {  in find_inode_fast()
950 			__wait_on_freeing_inode(inode);  in find_inode_fast()
953 		if (unlikely(inode->i_state & I_CREATING)) {  in find_inode_fast()
954 			spin_unlock(&inode->i_lock);  in find_inode_fast()
957 		__iget(inode);  in find_inode_fast()
958 		spin_unlock(&inode->i_lock);  in find_inode_fast()
959 		return inode;  in find_inode_fast()
1016 struct inode *new_inode_pseudo(struct super_block *sb)  in new_inode_pseudo()
1018 	struct inode *inode = alloc_inode(sb);  in new_inode_pseudo()  local
1020 	if (inode) {  in new_inode_pseudo()
1021 		spin_lock(&inode->i_lock);  in new_inode_pseudo()
1022 		inode->i_state = 0;  in new_inode_pseudo()
1023 		spin_unlock(&inode->i_lock);  in new_inode_pseudo()
1025 	return inode;  in new_inode_pseudo()
1040 struct inode *new_inode(struct super_block *sb)  in new_inode()
1042 	struct inode *inode;  in new_inode()  local
1046 	inode = new_inode_pseudo(sb);  in new_inode()
1047 	if (inode)  in new_inode()
1048 		inode_sb_list_add(inode);  in new_inode()
1049 	return inode;  in new_inode()
1054 void lockdep_annotate_inode_mutex_key(struct inode *inode)  in lockdep_annotate_inode_mutex_key()  argument
1056 	if (S_ISDIR(inode->i_mode)) {  in lockdep_annotate_inode_mutex_key()
1057 		struct file_system_type *type = inode->i_sb->s_type;  in lockdep_annotate_inode_mutex_key()
1060 		if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {  in lockdep_annotate_inode_mutex_key()
1065 			init_rwsem(&inode->i_rwsem);  in lockdep_annotate_inode_mutex_key()
1066 			lockdep_set_class(&inode->i_rwsem,  in lockdep_annotate_inode_mutex_key()
1081 void unlock_new_inode(struct inode *inode)  in unlock_new_inode()  argument
1083 	lockdep_annotate_inode_mutex_key(inode);  in unlock_new_inode()
1084 	spin_lock(&inode->i_lock);  in unlock_new_inode()
1085 	WARN_ON(!(inode->i_state & I_NEW));  in unlock_new_inode()
1086 	inode->i_state &= ~I_NEW & ~I_CREATING;  in unlock_new_inode()
1088 	wake_up_bit(&inode->i_state, __I_NEW);  in unlock_new_inode()
1089 	spin_unlock(&inode->i_lock);  in unlock_new_inode()
1093 void discard_new_inode(struct inode *inode)  in discard_new_inode()  argument
1095 	lockdep_annotate_inode_mutex_key(inode);  in discard_new_inode()
1096 	spin_lock(&inode->i_lock);  in discard_new_inode()
1097 	WARN_ON(!(inode->i_state & I_NEW));  in discard_new_inode()
1098 	inode->i_state &= ~I_NEW;  in discard_new_inode()
1100 	wake_up_bit(&inode->i_state, __I_NEW);  in discard_new_inode()
1101 	spin_unlock(&inode->i_lock);  in discard_new_inode()
1102 	iput(inode);  in discard_new_inode()
1115 void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)  in lock_two_nondirectories()
1132 void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)  in unlock_two_nondirectories()
1161 struct inode *inode_insert5(struct inode *inode, unsigned long hashval,  in inode_insert5()  argument
1162 			    int (*test)(struct inode *, void *),  in inode_insert5()  argument
1163 			    int (*set)(struct inode *, void *), void *data)  in inode_insert5()  argument
1165 	struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);  in inode_insert5()
1166 	struct inode *old;  in inode_insert5()
1170 	old = find_inode(inode->i_sb, head, test, data);  in inode_insert5()
1187 	if (set && unlikely(set(inode, data))) {  in inode_insert5()
1188 		inode = NULL;  in inode_insert5()
1196 	spin_lock(&inode->i_lock);  in inode_insert5()
1197 	inode->i_state |= I_NEW;  in inode_insert5()
1198 	hlist_add_head_rcu(&inode->i_hash, head);  in inode_insert5()
1199 	spin_unlock(&inode->i_lock);  in inode_insert5()
1205 	if (list_empty(&inode->i_sb_list))  in inode_insert5()
1206 		inode_sb_list_add(inode);  in inode_insert5()
1210 	return inode;  in inode_insert5()
1234 struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,  in iget5_locked()
1235 		int (*test)(struct inode *, void *),  in iget5_locked()  argument
1236 		int (*set)(struct inode *, void *), void *data)  in iget5_locked()  argument
1238 	struct inode *inode = ilookup5(sb, hashval, test, data);  in iget5_locked()  local
1240 	if (!inode) {  in iget5_locked()
1241 		struct inode *new = alloc_inode(sb);  in iget5_locked()
1245 			inode = inode_insert5(new, hashval, test, set, data);  in iget5_locked()
1246 			if (unlikely(inode != new))  in iget5_locked()
1250 	return inode;  in iget5_locked()
1267 struct inode *iget_locked(struct super_block *sb, unsigned long ino)  in iget_locked()
1270 	struct inode *inode;  in iget_locked()  local
1273 	inode = find_inode_fast(sb, head, ino);  in iget_locked()
1275 	if (inode) {  in iget_locked()
1276 		if (IS_ERR(inode))  in iget_locked()
1278 		wait_on_inode(inode);  in iget_locked()
1279 		if (unlikely(inode_unhashed(inode))) {  in iget_locked()
1280 			iput(inode);  in iget_locked()
1283 		return inode;  in iget_locked()
1286 	inode = alloc_inode(sb);  in iget_locked()
1287 	if (inode) {  in iget_locked()
1288 		struct inode *old;  in iget_locked()
1294 			inode->i_ino = ino;  in iget_locked()
1295 			spin_lock(&inode->i_lock);  in iget_locked()
1296 			inode->i_state = I_NEW;  in iget_locked()
1297 			hlist_add_head_rcu(&inode->i_hash, head);  in iget_locked()
1298 			spin_unlock(&inode->i_lock);  in iget_locked()
1299 			inode_sb_list_add(inode);  in iget_locked()
1305 			return inode;  in iget_locked()
1314 		destroy_inode(inode);  in iget_locked()
1317 		inode = old;  in iget_locked()
1318 		wait_on_inode(inode);  in iget_locked()
1319 		if (unlikely(inode_unhashed(inode))) {  in iget_locked()
1320 			iput(inode);  in iget_locked()
1324 	return inode;  in iget_locked()
1338 	struct inode *inode;  in test_inode_iunique()  local
1340 	hlist_for_each_entry_rcu(inode, b, i_hash) {  in test_inode_iunique()
1341 		if (inode->i_ino == ino && inode->i_sb == sb)  in test_inode_iunique()
1386 struct inode *igrab(struct inode *inode)  in igrab()  argument
1388 	spin_lock(&inode->i_lock);  in igrab()
1389 	if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {  in igrab()
1390 		__iget(inode);  in igrab()
1391 		spin_unlock(&inode->i_lock);  in igrab()
1393 		spin_unlock(&inode->i_lock);  in igrab()
1399 		inode = NULL;  in igrab()
1401 	return inode;  in igrab()
1421 struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,  in ilookup5_nowait()
1422 		int (*test)(struct inode *, void *), void *data)  in ilookup5_nowait()  argument
1425 	struct inode *inode;  in ilookup5_nowait()  local
1428 	inode = find_inode(sb, head, test, data);  in ilookup5_nowait()
1431 	return IS_ERR(inode) ? NULL : inode;  in ilookup5_nowait()
1452 struct inode *ilookup5(struct super_block *sb, unsigned long hashval,  in ilookup5()
1453 		int (*test)(struct inode *, void *), void *data)  in ilookup5()  argument
1455 	struct inode *inode;  in ilookup5()  local
1457 	inode = ilookup5_nowait(sb, hashval, test, data);  in ilookup5()
1458 	if (inode) {  in ilookup5()
1459 		wait_on_inode(inode);  in ilookup5()
1460 		if (unlikely(inode_unhashed(inode))) {  in ilookup5()
1461 			iput(inode);  in ilookup5()
1465 	return inode;  in ilookup5()
1477 struct inode *ilookup(struct super_block *sb, unsigned long ino)  in ilookup()
1480 	struct inode *inode;  in ilookup()  local
1483 	inode = find_inode_fast(sb, head, ino);  in ilookup()
1486 	if (inode) {  in ilookup()
1487 		if (IS_ERR(inode))  in ilookup()
1489 		wait_on_inode(inode);  in ilookup()
1490 		if (unlikely(inode_unhashed(inode))) {  in ilookup()
1491 			iput(inode);  in ilookup()
1495 	return inode;  in ilookup()
1522 struct inode *find_inode_nowait(struct super_block *sb,  in find_inode_nowait()
1524 				int (*match)(struct inode *, unsigned long,  in find_inode_nowait()  argument
1529 	struct inode *inode, *ret_inode = NULL;  in find_inode_nowait()  local
1533 	hlist_for_each_entry(inode, head, i_hash) {  in find_inode_nowait()
1534 		if (inode->i_sb != sb)  in find_inode_nowait()
1536 		mval = match(inode, hashval, data);  in find_inode_nowait()
1540 			ret_inode = inode;  in find_inode_nowait()
1570 struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,  in find_inode_rcu()
1571 			     int (*test)(struct inode *, void *), void *data)  in find_inode_rcu()  argument
1574 	struct inode *inode;  in find_inode_rcu()  local
1579 	hlist_for_each_entry_rcu(inode, head, i_hash) {  in find_inode_rcu()
1580 		if (inode->i_sb == sb &&  in find_inode_rcu()
1581 		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) &&  in find_inode_rcu()
1582 		    test(inode, data))  in find_inode_rcu()
1583 			return inode;  in find_inode_rcu()
1608 struct inode *find_inode_by_ino_rcu(struct super_block *sb,  in find_inode_by_ino_rcu()
1612 	struct inode *inode;  in find_inode_by_ino_rcu()  local
1617 	hlist_for_each_entry_rcu(inode, head, i_hash) {  in find_inode_by_ino_rcu()
1618 		if (inode->i_ino == ino &&  in find_inode_by_ino_rcu()
1619 		    inode->i_sb == sb &&  in find_inode_by_ino_rcu()
1620 		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)))  in find_inode_by_ino_rcu()
1621 		    return inode;  in find_inode_by_ino_rcu()
1627 int insert_inode_locked(struct inode *inode)  in insert_inode_locked()  argument
1629 	struct super_block *sb = inode->i_sb;  in insert_inode_locked()
1630 	ino_t ino = inode->i_ino;  in insert_inode_locked()
1634 		struct inode *old = NULL;  in insert_inode_locked()
1649 			spin_lock(&inode->i_lock);  in insert_inode_locked()
1650 			inode->i_state |= I_NEW | I_CREATING;  in insert_inode_locked()
1651 			hlist_add_head_rcu(&inode->i_hash, head);  in insert_inode_locked()
1652 			spin_unlock(&inode->i_lock);  in insert_inode_locked()
1674 int insert_inode_locked4(struct inode *inode, unsigned long hashval,  in insert_inode_locked4()  argument
1675 		int (*test)(struct inode *, void *), void *data)  in insert_inode_locked4()  argument
1677 	struct inode *old;  in insert_inode_locked4()
1679 	inode->i_state |= I_CREATING;  in insert_inode_locked4()
1680 	old = inode_insert5(inode, hashval, test, NULL, data);  in insert_inode_locked4()
1682 	if (old != inode) {  in insert_inode_locked4()
1691 int generic_delete_inode(struct inode *inode)  in generic_delete_inode()  argument
1707 static void iput_final(struct inode *inode)  in iput_final()  argument
1709 	struct super_block *sb = inode->i_sb;  in iput_final()
1710 	const struct super_operations *op = inode->i_sb->s_op;  in iput_final()
1714 	WARN_ON(inode->i_state & I_NEW);  in iput_final()
1717 		drop = op->drop_inode(inode);  in iput_final()
1719 		drop = generic_drop_inode(inode);  in iput_final()
1722 	    !(inode->i_state & I_DONTCACHE) &&  in iput_final()
1724 		__inode_add_lru(inode, true);  in iput_final()
1725 		spin_unlock(&inode->i_lock);  in iput_final()
1729 	state = inode->i_state;  in iput_final()
1731 		WRITE_ONCE(inode->i_state, state | I_WILL_FREE);  in iput_final()
1732 		spin_unlock(&inode->i_lock);  in iput_final()
1734 		write_inode_now(inode, 1);  in iput_final()
1736 		spin_lock(&inode->i_lock);  in iput_final()
1737 		state = inode->i_state;  in iput_final()
1742 	WRITE_ONCE(inode->i_state, state | I_FREEING);  in iput_final()
1743 	if (!list_empty(&inode->i_lru))  in iput_final()
1744 		inode_lru_list_del(inode);  in iput_final()
1745 	spin_unlock(&inode->i_lock);  in iput_final()
1747 	evict(inode);  in iput_final()
1759 void iput(struct inode *inode)  in iput()  argument
1761 	if (!inode)  in iput()
1763 	BUG_ON(inode->i_state & I_CLEAR);  in iput()
1765 	if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {  in iput()
1766 		if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {  in iput()
1767 			atomic_inc(&inode->i_count);  in iput()
1768 			spin_unlock(&inode->i_lock);  in iput()
1769 			trace_writeback_lazytime_iput(inode);  in iput()
1770 			mark_inode_dirty_sync(inode);  in iput()
1773 		iput_final(inode);  in iput()
1793 int bmap(struct inode *inode, sector_t *block)  in bmap()  argument
1795 	if (!inode->i_mapping->a_ops->bmap)  in bmap()
1798 	*block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block);  in bmap()
1809 static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,  in relatime_need_update()  argument
1818 	if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)  in relatime_need_update()
1823 	if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)  in relatime_need_update()
1830 	if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)  in relatime_need_update()
1838 int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)  in generic_update_time()  argument
1844 			inode->i_atime = *time;  in generic_update_time()
1846 			inode->i_ctime = *time;  in generic_update_time()
1848 			inode->i_mtime = *time;  in generic_update_time()
1850 		if (inode->i_sb->s_flags & SB_LAZYTIME)  in generic_update_time()
1856 	if ((flags & S_VERSION) && inode_maybe_inc_iversion(inode, false))  in generic_update_time()
1859 	__mark_inode_dirty(inode, dirty_flags);  in generic_update_time()
1868 int inode_update_time(struct inode *inode, struct timespec64 *time, int flags)  in inode_update_time()  argument
1870 	if (inode->i_op->update_time)  in inode_update_time()
1871 		return inode->i_op->update_time(inode, time, flags);  in inode_update_time()
1872 	return generic_update_time(inode, time, flags);  in inode_update_time()
1885 bool atime_needs_update(const struct path *path, struct inode *inode)  in atime_needs_update()  argument
1890 	if (inode->i_flags & S_NOATIME)  in atime_needs_update()
1896 	if (HAS_UNMAPPED_ID(mnt_user_ns(mnt), inode))  in atime_needs_update()
1899 	if (IS_NOATIME(inode))  in atime_needs_update()
1901 	if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))  in atime_needs_update()
1906 	if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))  in atime_needs_update()
1909 	now = current_time(inode);  in atime_needs_update()
1911 	if (!relatime_need_update(mnt, inode, now))  in atime_needs_update()
1914 	if (timespec64_equal(&inode->i_atime, &now))  in atime_needs_update()
1923 	struct inode *inode = d_inode(path->dentry);  in touch_atime()  local
1926 	if (!atime_needs_update(path, inode))  in touch_atime()
1929 	if (!sb_start_write_trylock(inode->i_sb))  in touch_atime()
1943 	now = current_time(inode);  in touch_atime()
1944 	inode_update_time(inode, &now, S_ATIME);  in touch_atime()
1947 	sb_end_write(inode->i_sb);  in touch_atime()
1987 	struct inode *inode = d_inode(dentry);  in dentry_needs_remove_privs()  local
1991 	if (IS_NOSEC(inode))  in dentry_needs_remove_privs()
2019 	struct inode *inode = file_inode(file);  in __file_remove_privs()  local
2023 	if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))  in __file_remove_privs()
2038 		inode_has_no_xattr(inode);  in __file_remove_privs()
2057 static int inode_needs_update_time(struct inode *inode, struct timespec64 *now)  in inode_needs_update_time()  argument
2062 	if (IS_NOCMTIME(inode))  in inode_needs_update_time()
2065 	if (!timespec64_equal(&inode->i_mtime, now))  in inode_needs_update_time()
2068 	if (!timespec64_equal(&inode->i_ctime, now))  in inode_needs_update_time()
2071 	if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))  in inode_needs_update_time()
2084 	struct inode *inode = file_inode(file);  in __file_update_time()  local
2088 		ret = inode_update_time(inode, now, sync_mode);  in __file_update_time()
2112 	struct inode *inode = file_inode(file);  in file_update_time()  local
2113 	struct timespec64 now = current_time(inode);  in file_update_time()
2115 	ret = inode_needs_update_time(inode, &now);  in file_update_time()
2141 	struct inode *inode = file_inode(file);  in file_modified_flags()  local
2142 	struct timespec64 now = current_time(inode);  in file_modified_flags()
2155 	ret = inode_needs_update_time(inode, &now);  in file_modified_flags()
2198 int inode_needs_sync(struct inode *inode)  in inode_needs_sync()  argument
2200 	if (IS_SYNC(inode))  in inode_needs_sync()
2202 	if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))  in inode_needs_sync()
2219 static void __wait_on_freeing_inode(struct inode *inode)  in __wait_on_freeing_inode()  argument
2222 	DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);  in __wait_on_freeing_inode()
2223 	wq = bit_waitqueue(&inode->i_state, __I_NEW);  in __wait_on_freeing_inode()
2225 	spin_unlock(&inode->i_lock);  in __wait_on_freeing_inode()
2269 					 sizeof(struct inode),  in inode_init()
2291 void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)  in init_special_inode()  argument
2293 	inode->i_mode = mode;  in init_special_inode()
2295 		inode->i_fop = &def_chr_fops;  in init_special_inode()
2296 		inode->i_rdev = rdev;  in init_special_inode()
2298 		inode->i_fop = &def_blk_fops;  in init_special_inode()
2299 		inode->i_rdev = rdev;  in init_special_inode()
2301 		inode->i_fop = &pipefifo_fops;  in init_special_inode()
2306 				  " inode %s:%lu\n", mode, inode->i_sb->s_id,  in init_special_inode()
2307 				  inode->i_ino);  in init_special_inode()
2324 void inode_init_owner(struct user_namespace *mnt_userns, struct inode *inode,  in inode_init_owner()  argument
2325 		      const struct inode *dir, umode_t mode)  in inode_init_owner()
2327 	inode_fsuid_set(inode, mnt_userns);  in inode_init_owner()
2329 		inode->i_gid = dir->i_gid;  in inode_init_owner()
2335 		inode_fsgid_set(inode, mnt_userns);  in inode_init_owner()
2336 	inode->i_mode = mode;  in inode_init_owner()
2355 			    const struct inode *inode)  in inode_owner_or_capable()  argument
2360 	i_uid = i_uid_into_mnt(mnt_userns, inode);  in inode_owner_or_capable()
2374 static void __inode_dio_wait(struct inode *inode)  in __inode_dio_wait()  argument
2376 	wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);  in __inode_dio_wait()
2377 	DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);  in __inode_dio_wait()
2381 		if (atomic_read(&inode->i_dio_count))  in __inode_dio_wait()
2383 	} while (atomic_read(&inode->i_dio_count));  in __inode_dio_wait()
2397 void inode_dio_wait(struct inode *inode)  in inode_dio_wait()  argument
2399 	if (atomic_read(&inode->i_dio_count))  in inode_dio_wait()
2400 		__inode_dio_wait(inode);  in inode_dio_wait()
2420 void inode_set_flags(struct inode *inode, unsigned int flags,  in inode_set_flags()  argument
2424 	set_mask_bits(&inode->i_flags, mask, flags);  in inode_set_flags()
2428 void inode_nohighmem(struct inode *inode)  in inode_nohighmem()  argument
2430 	mapping_set_gfp_mask(inode->i_mapping, GFP_USER);  in inode_nohighmem()
2443 struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)  in timestamp_truncate()  argument
2445 	struct super_block *sb = inode->i_sb;  in timestamp_truncate()
2475 struct timespec64 current_time(struct inode *inode)  in current_time()  argument
2481 	if (unlikely(!inode->i_sb)) {  in current_time()
2486 	return timestamp_truncate(now, inode);  in current_time()
2505 			const struct inode *dir, umode_t mode)  in mode_strip_sgid()