143 static int no_open(struct inode *inode, struct file *file)  in no_open()  argument
156 int inode_init_always(struct super_block *sb, struct inode *inode)  in inode_init_always()  argument
160 	struct address_space *const mapping = &inode->i_data;  in inode_init_always()
162 	inode->i_sb = sb;  in inode_init_always()
163 	inode->i_blkbits = sb->s_blocksize_bits;  in inode_init_always()
164 	inode->i_flags = 0;  in inode_init_always()
165 	atomic64_set(&inode->i_sequence, 0);  in inode_init_always()
166 	atomic_set(&inode->i_count, 1);  in inode_init_always()
167 	inode->i_op = &empty_iops;  in inode_init_always()
168 	inode->i_fop = &no_open_fops;  in inode_init_always()
169 	inode->i_ino = 0;  in inode_init_always()
170 	inode->__i_nlink = 1;  in inode_init_always()
171 	inode->i_opflags = 0;  in inode_init_always()
173 		inode->i_opflags |= IOP_XATTR;  in inode_init_always()
174 	i_uid_write(inode, 0);  in inode_init_always()
175 	i_gid_write(inode, 0);  in inode_init_always()
176 	atomic_set(&inode->i_writecount, 0);  in inode_init_always()
177 	inode->i_size = 0;  in inode_init_always()
178 	inode->i_write_hint = WRITE_LIFE_NOT_SET;  in inode_init_always()
179 	inode->i_blocks = 0;  in inode_init_always()
180 	inode->i_bytes = 0;  in inode_init_always()
181 	inode->i_generation = 0;  in inode_init_always()
182 	inode->i_pipe = NULL;  in inode_init_always()
183 	inode->i_cdev = NULL;  in inode_init_always()
184 	inode->i_link = NULL;  in inode_init_always()
185 	inode->i_dir_seq = 0;  in inode_init_always()
186 	inode->i_rdev = 0;  in inode_init_always()
187 	inode->dirtied_when = 0;  in inode_init_always()
190 	inode->i_wb_frn_winner = 0;  in inode_init_always()
191 	inode->i_wb_frn_avg_time = 0;  in inode_init_always()
192 	inode->i_wb_frn_history = 0;  in inode_init_always()
195 	spin_lock_init(&inode->i_lock);  in inode_init_always()
196 	lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);  in inode_init_always()
198 	init_rwsem(&inode->i_rwsem);  in inode_init_always()
199 	lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);  in inode_init_always()
201 	atomic_set(&inode->i_dio_count, 0);  in inode_init_always()
204 	mapping->host = inode;  in inode_init_always()
218 	inode->i_private = NULL;  in inode_init_always()
219 	inode->i_mapping = mapping;  in inode_init_always()
220 	INIT_HLIST_HEAD(&inode->i_dentry);	/* buggered by rcu freeing */  in inode_init_always()
222 	inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;  in inode_init_always()
226 	inode->i_fsnotify_mask = 0;  in inode_init_always()
228 	inode->i_flctx = NULL;  in inode_init_always()
230 	if (unlikely(security_inode_alloc(inode)))  in inode_init_always()
238 void free_inode_nonrcu(struct inode *inode)  in free_inode_nonrcu()  argument
240 	kmem_cache_free(inode_cachep, inode);  in free_inode_nonrcu()
246 	struct inode *inode = container_of(head, struct inode, i_rcu);  in i_callback()  local
247 	if (inode->free_inode)  in i_callback()
248 		inode->free_inode(inode);  in i_callback()
250 		free_inode_nonrcu(inode);  in i_callback()
253 static struct inode *alloc_inode(struct super_block *sb)  in alloc_inode()
256 	struct inode *inode;  in alloc_inode()  local
259 		inode = ops->alloc_inode(sb);  in alloc_inode()
261 		inode = alloc_inode_sb(sb, inode_cachep, GFP_KERNEL);  in alloc_inode()
263 	if (!inode)  in alloc_inode()
266 	if (unlikely(inode_init_always(sb, inode))) {  in alloc_inode()
268 			ops->destroy_inode(inode);  in alloc_inode()
272 		inode->free_inode = ops->free_inode;  in alloc_inode()
273 		i_callback(&inode->i_rcu);  in alloc_inode()
277 	return inode;  in alloc_inode()
280 void __destroy_inode(struct inode *inode)  in __destroy_inode()  argument
282 	BUG_ON(inode_has_buffers(inode));  in __destroy_inode()
283 	inode_detach_wb(inode);  in __destroy_inode()
284 	security_inode_free(inode);  in __destroy_inode()
285 	fsnotify_inode_delete(inode);  in __destroy_inode()
286 	locks_free_lock_context(inode);  in __destroy_inode()
287 	if (!inode->i_nlink) {  in __destroy_inode()
288 		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);  in __destroy_inode()
289 		atomic_long_dec(&inode->i_sb->s_remove_count);  in __destroy_inode()
293 	if (inode->i_acl && !is_uncached_acl(inode->i_acl))  in __destroy_inode()
294 		posix_acl_release(inode->i_acl);  in __destroy_inode()
295 	if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))  in __destroy_inode()
296 		posix_acl_release(inode->i_default_acl);  in __destroy_inode()
302 static void destroy_inode(struct inode *inode)  in destroy_inode()  argument
304 	const struct super_operations *ops = inode->i_sb->s_op;  in destroy_inode()
306 	BUG_ON(!list_empty(&inode->i_lru));  in destroy_inode()
307 	__destroy_inode(inode);  in destroy_inode()
309 		ops->destroy_inode(inode);  in destroy_inode()
313 	inode->free_inode = ops->free_inode;  in destroy_inode()
314 	call_rcu(&inode->i_rcu, i_callback);  in destroy_inode()
328 void drop_nlink(struct inode *inode)  in drop_nlink()  argument
330 	WARN_ON(inode->i_nlink == 0);  in drop_nlink()
331 	inode->__i_nlink--;  in drop_nlink()
332 	if (!inode->i_nlink)  in drop_nlink()
333 		atomic_long_inc(&inode->i_sb->s_remove_count);  in drop_nlink()
345 void clear_nlink(struct inode *inode)  in clear_nlink()  argument
347 	if (inode->i_nlink) {  in clear_nlink()
348 		inode->__i_nlink = 0;  in clear_nlink()
349 		atomic_long_inc(&inode->i_sb->s_remove_count);  in clear_nlink()
362 void set_nlink(struct inode *inode, unsigned int nlink)  in set_nlink()  argument
365 		clear_nlink(inode);  in set_nlink()
368 		if (inode->i_nlink == 0)  in set_nlink()
369 			atomic_long_dec(&inode->i_sb->s_remove_count);  in set_nlink()
371 		inode->__i_nlink = nlink;  in set_nlink()
384 void inc_nlink(struct inode *inode)  in inc_nlink()  argument
386 	if (unlikely(inode->i_nlink == 0)) {  in inc_nlink()
387 		WARN_ON(!(inode->i_state & I_LINKABLE));  in inc_nlink()
388 		atomic_long_dec(&inode->i_sb->s_remove_count);  in inc_nlink()
391 	inode->__i_nlink++;  in inc_nlink()
416 void inode_init_once(struct inode *inode)  in inode_init_once()  argument
418 	memset(inode, 0, sizeof(*inode));  in inode_init_once()
419 	INIT_HLIST_NODE(&inode->i_hash);  in inode_init_once()
420 	INIT_LIST_HEAD(&inode->i_devices);  in inode_init_once()
421 	INIT_LIST_HEAD(&inode->i_io_list);  in inode_init_once()
422 	INIT_LIST_HEAD(&inode->i_wb_list);  in inode_init_once()
423 	INIT_LIST_HEAD(&inode->i_lru);  in inode_init_once()
424 	INIT_LIST_HEAD(&inode->i_sb_list);  in inode_init_once()
425 	__address_space_init_once(&inode->i_data);  in inode_init_once()
426 	i_size_ordered_init(inode);  in inode_init_once()
432 	struct inode *inode = (struct inode *) foo;  in init_once()  local
434 	inode_init_once(inode);  in init_once()
440 void __iget(struct inode *inode)  in __iget()  argument
442 	atomic_inc(&inode->i_count);  in __iget()
448 void ihold(struct inode *inode)  in ihold()  argument
450 	WARN_ON(atomic_inc_return(&inode->i_count) < 2);  in ihold()
454 static void __inode_add_lru(struct inode *inode, bool rotate)  in __inode_add_lru()  argument
456 	if (inode->i_state & (I_DIRTY_ALL | I_SYNC | I_FREEING | I_WILL_FREE))  in __inode_add_lru()
458 	if (atomic_read(&inode->i_count))  in __inode_add_lru()
460 	if (!(inode->i_sb->s_flags & SB_ACTIVE))  in __inode_add_lru()
462 	if (!mapping_shrinkable(&inode->i_data))  in __inode_add_lru()
465 	if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))  in __inode_add_lru()
468 		inode->i_state |= I_REFERENCED;  in __inode_add_lru()
476 void inode_add_lru(struct inode *inode)  in inode_add_lru()  argument
478 	__inode_add_lru(inode, false);  in inode_add_lru()
481 static void inode_lru_list_del(struct inode *inode)  in inode_lru_list_del()  argument
483 	if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))  in inode_lru_list_del()
491 void inode_sb_list_add(struct inode *inode)  in inode_sb_list_add()  argument
493 	spin_lock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_add()
494 	list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);  in inode_sb_list_add()
495 	spin_unlock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_add()
499 static inline void inode_sb_list_del(struct inode *inode)  in inode_sb_list_del()  argument
501 	if (!list_empty(&inode->i_sb_list)) {  in inode_sb_list_del()
502 		spin_lock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_del()
503 		list_del_init(&inode->i_sb_list);  in inode_sb_list_del()
504 		spin_unlock(&inode->i_sb->s_inode_list_lock);  in inode_sb_list_del()
526 void __insert_inode_hash(struct inode *inode, unsigned long hashval)  in __insert_inode_hash()  argument
528 	struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);  in __insert_inode_hash()
531 	spin_lock(&inode->i_lock);  in __insert_inode_hash()
532 	hlist_add_head_rcu(&inode->i_hash, b);  in __insert_inode_hash()
533 	spin_unlock(&inode->i_lock);  in __insert_inode_hash()
544 void __remove_inode_hash(struct inode *inode)  in __remove_inode_hash()  argument
547 	spin_lock(&inode->i_lock);  in __remove_inode_hash()
548 	hlist_del_init_rcu(&inode->i_hash);  in __remove_inode_hash()
549 	spin_unlock(&inode->i_lock);  in __remove_inode_hash()
556 	struct inode *host;  in dump_mapping()
603 void clear_inode(struct inode *inode)  in clear_inode()  argument
610 	xa_lock_irq(&inode->i_data.i_pages);  in clear_inode()
611 	BUG_ON(inode->i_data.nrpages);  in clear_inode()
620 	xa_unlock_irq(&inode->i_data.i_pages);  in clear_inode()
621 	BUG_ON(!list_empty(&inode->i_data.private_list));  in clear_inode()
622 	BUG_ON(!(inode->i_state & I_FREEING));  in clear_inode()
623 	BUG_ON(inode->i_state & I_CLEAR);  in clear_inode()
624 	BUG_ON(!list_empty(&inode->i_wb_list));  in clear_inode()
626 	inode->i_state = I_FREEING | I_CLEAR;  in clear_inode()
643 static void evict(struct inode *inode)  in evict()  argument
645 	const struct super_operations *op = inode->i_sb->s_op;  in evict()
647 	BUG_ON(!(inode->i_state & I_FREEING));  in evict()
648 	BUG_ON(!list_empty(&inode->i_lru));  in evict()
650 	if (!list_empty(&inode->i_io_list))  in evict()
651 		inode_io_list_del(inode);  in evict()
653 	inode_sb_list_del(inode);  in evict()
661 	inode_wait_for_writeback(inode);  in evict()
664 		op->evict_inode(inode);  in evict()
666 		truncate_inode_pages_final(&inode->i_data);  in evict()
667 		clear_inode(inode);  in evict()
669 	if (S_ISCHR(inode->i_mode) && inode->i_cdev)  in evict()
670 		cd_forget(inode);  in evict()
672 	remove_inode_hash(inode);  in evict()
674 	spin_lock(&inode->i_lock);  in evict()
675 	wake_up_bit(&inode->i_state, __I_NEW);  in evict()
676 	BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));  in evict()
677 	spin_unlock(&inode->i_lock);  in evict()
679 	destroy_inode(inode);  in evict()
692 		struct inode *inode;  in dispose_list()  local
694 		inode = list_first_entry(head, struct inode, i_lru);  in dispose_list()
695 		list_del_init(&inode->i_lru);  in dispose_list()
697 		evict(inode);  in dispose_list()
713 	struct inode *inode, *next;  in evict_inodes()  local
718 	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {  in evict_inodes()
719 		if (atomic_read(&inode->i_count))  in evict_inodes()
722 		spin_lock(&inode->i_lock);  in evict_inodes()
723 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {  in evict_inodes()
724 			spin_unlock(&inode->i_lock);  in evict_inodes()
728 		inode->i_state |= I_FREEING;  in evict_inodes()
729 		inode_lru_list_del(inode);  in evict_inodes()
730 		spin_unlock(&inode->i_lock);  in evict_inodes()
731 		list_add(&inode->i_lru, &dispose);  in evict_inodes()
759 	struct inode *inode, *next;  in invalidate_inodes()  local
764 	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {  in invalidate_inodes()
765 		spin_lock(&inode->i_lock);  in invalidate_inodes()
766 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {  in invalidate_inodes()
767 			spin_unlock(&inode->i_lock);  in invalidate_inodes()
770 		if (atomic_read(&inode->i_count)) {  in invalidate_inodes()
771 			spin_unlock(&inode->i_lock);  in invalidate_inodes()
775 		inode->i_state |= I_FREEING;  in invalidate_inodes()
776 		inode_lru_list_del(inode);  in invalidate_inodes()
777 		spin_unlock(&inode->i_lock);  in invalidate_inodes()
778 		list_add(&inode->i_lru, &dispose);  in invalidate_inodes()
806 	struct inode	*inode = container_of(item, struct inode, i_lru);  in inode_lru_isolate()  local
812 	if (!spin_trylock(&inode->i_lock))  in inode_lru_isolate()
821 	if (atomic_read(&inode->i_count) ||  in inode_lru_isolate()
822 	    (inode->i_state & ~I_REFERENCED) ||  in inode_lru_isolate()
823 	    !mapping_shrinkable(&inode->i_data)) {  in inode_lru_isolate()
824 		list_lru_isolate(lru, &inode->i_lru);  in inode_lru_isolate()
825 		spin_unlock(&inode->i_lock);  in inode_lru_isolate()
831 	if (inode->i_state & I_REFERENCED) {  in inode_lru_isolate()
832 		inode->i_state &= ~I_REFERENCED;  in inode_lru_isolate()
833 		spin_unlock(&inode->i_lock);  in inode_lru_isolate()
842 	if (inode_has_buffers(inode) || !mapping_empty(&inode->i_data)) {  in inode_lru_isolate()
843 		__iget(inode);  in inode_lru_isolate()
844 		spin_unlock(&inode->i_lock);  in inode_lru_isolate()
846 		if (remove_inode_buffers(inode)) {  in inode_lru_isolate()
848 			reap = invalidate_mapping_pages(&inode->i_data, 0, -1);  in inode_lru_isolate()
855 		iput(inode);  in inode_lru_isolate()
860 	WARN_ON(inode->i_state & I_NEW);  in inode_lru_isolate()
861 	inode->i_state |= I_FREEING;  in inode_lru_isolate()
862 	list_lru_isolate_move(lru, &inode->i_lru, freeable);  in inode_lru_isolate()
863 	spin_unlock(&inode->i_lock);  in inode_lru_isolate()
886 static void __wait_on_freeing_inode(struct inode *inode);
890 static struct inode *find_inode(struct super_block *sb,  in find_inode()
892 				int (*test)(struct inode *, void *),  in find_inode()  argument
895 	struct inode *inode = NULL;  in find_inode()  local
898 	hlist_for_each_entry(inode, head, i_hash) {  in find_inode()
899 		if (inode->i_sb != sb)  in find_inode()
901 		if (!test(inode, data))  in find_inode()
903 		spin_lock(&inode->i_lock);  in find_inode()
904 		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {  in find_inode()
905 			__wait_on_freeing_inode(inode);  in find_inode()
908 		if (unlikely(inode->i_state & I_CREATING)) {  in find_inode()
909 			spin_unlock(&inode->i_lock);  in find_inode()
912 		__iget(inode);  in find_inode()
913 		spin_unlock(&inode->i_lock);  in find_inode()
914 		return inode;  in find_inode()
923 static struct inode *find_inode_fast(struct super_block *sb,  in find_inode_fast()
926 	struct inode *inode = NULL;  in find_inode_fast()  local
929 	hlist_for_each_entry(inode, head, i_hash) {  in find_inode_fast()
930 		if (inode->i_ino != ino)  in find_inode_fast()
932 		if (inode->i_sb != sb)  in find_inode_fast()
934 		spin_lock(&inode->i_lock);  in find_inode_fast()
935 		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {  in find_inode_fast()
936 			__wait_on_freeing_inode(inode);  in find_inode_fast()
939 		if (unlikely(inode->i_state & I_CREATING)) {  in find_inode_fast()
940 			spin_unlock(&inode->i_lock);  in find_inode_fast()
943 		__iget(inode);  in find_inode_fast()
944 		spin_unlock(&inode->i_lock);  in find_inode_fast()
945 		return inode;  in find_inode_fast()
1002 struct inode *new_inode_pseudo(struct super_block *sb)  in new_inode_pseudo()
1004 	struct inode *inode = alloc_inode(sb);  in new_inode_pseudo()  local
1006 	if (inode) {  in new_inode_pseudo()
1007 		spin_lock(&inode->i_lock);  in new_inode_pseudo()
1008 		inode->i_state = 0;  in new_inode_pseudo()
1009 		spin_unlock(&inode->i_lock);  in new_inode_pseudo()
1011 	return inode;  in new_inode_pseudo()
1026 struct inode *new_inode(struct super_block *sb)  in new_inode()
1028 	struct inode *inode;  in new_inode()  local
1030 	inode = new_inode_pseudo(sb);  in new_inode()
1031 	if (inode)  in new_inode()
1032 		inode_sb_list_add(inode);  in new_inode()
1033 	return inode;  in new_inode()
1038 void lockdep_annotate_inode_mutex_key(struct inode *inode)  in lockdep_annotate_inode_mutex_key()  argument
1040 	if (S_ISDIR(inode->i_mode)) {  in lockdep_annotate_inode_mutex_key()
1041 		struct file_system_type *type = inode->i_sb->s_type;  in lockdep_annotate_inode_mutex_key()
1044 		if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {  in lockdep_annotate_inode_mutex_key()
1049 			init_rwsem(&inode->i_rwsem);  in lockdep_annotate_inode_mutex_key()
1050 			lockdep_set_class(&inode->i_rwsem,  in lockdep_annotate_inode_mutex_key()
1065 void unlock_new_inode(struct inode *inode)  in unlock_new_inode()  argument
1067 	lockdep_annotate_inode_mutex_key(inode);  in unlock_new_inode()
1068 	spin_lock(&inode->i_lock);  in unlock_new_inode()
1069 	WARN_ON(!(inode->i_state & I_NEW));  in unlock_new_inode()
1070 	inode->i_state &= ~I_NEW & ~I_CREATING;  in unlock_new_inode()
1072 	wake_up_bit(&inode->i_state, __I_NEW);  in unlock_new_inode()
1073 	spin_unlock(&inode->i_lock);  in unlock_new_inode()
1077 void discard_new_inode(struct inode *inode)  in discard_new_inode()  argument
1079 	lockdep_annotate_inode_mutex_key(inode);  in discard_new_inode()
1080 	spin_lock(&inode->i_lock);  in discard_new_inode()
1081 	WARN_ON(!(inode->i_state & I_NEW));  in discard_new_inode()
1082 	inode->i_state &= ~I_NEW;  in discard_new_inode()
1084 	wake_up_bit(&inode->i_state, __I_NEW);  in discard_new_inode()
1085 	spin_unlock(&inode->i_lock);  in discard_new_inode()
1086 	iput(inode);  in discard_new_inode()
1102 void lock_two_inodes(struct inode *inode1, struct inode *inode2,  in lock_two_inodes()
1141 void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)  in lock_two_nondirectories()
1156 void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)  in unlock_two_nondirectories()
1189 struct inode *inode_insert5(struct inode *inode, unsigned long hashval,  in inode_insert5()  argument
1190 			    int (*test)(struct inode *, void *),  in inode_insert5()  argument
1191 			    int (*set)(struct inode *, void *), void *data)  in inode_insert5()  argument
1193 	struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);  in inode_insert5()
1194 	struct inode *old;  in inode_insert5()
1198 	old = find_inode(inode->i_sb, head, test, data);  in inode_insert5()
1215 	if (set && unlikely(set(inode, data))) {  in inode_insert5()
1216 		inode = NULL;  in inode_insert5()
1224 	spin_lock(&inode->i_lock);  in inode_insert5()
1225 	inode->i_state |= I_NEW;  in inode_insert5()
1226 	hlist_add_head_rcu(&inode->i_hash, head);  in inode_insert5()
1227 	spin_unlock(&inode->i_lock);  in inode_insert5()
1233 	if (list_empty(&inode->i_sb_list))  in inode_insert5()
1234 		inode_sb_list_add(inode);  in inode_insert5()
1238 	return inode;  in inode_insert5()
1262 struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,  in iget5_locked()
1263 		int (*test)(struct inode *, void *),  in iget5_locked()  argument
1264 		int (*set)(struct inode *, void *), void *data)  in iget5_locked()  argument
1266 	struct inode *inode = ilookup5(sb, hashval, test, data);  in iget5_locked()  local
1268 	if (!inode) {  in iget5_locked()
1269 		struct inode *new = alloc_inode(sb);  in iget5_locked()
1273 			inode = inode_insert5(new, hashval, test, set, data);  in iget5_locked()
1274 			if (unlikely(inode != new))  in iget5_locked()
1278 	return inode;  in iget5_locked()
1295 struct inode *iget_locked(struct super_block *sb, unsigned long ino)  in iget_locked()
1298 	struct inode *inode;  in iget_locked()  local
1301 	inode = find_inode_fast(sb, head, ino);  in iget_locked()
1303 	if (inode) {  in iget_locked()
1304 		if (IS_ERR(inode))  in iget_locked()
1306 		wait_on_inode(inode);  in iget_locked()
1307 		if (unlikely(inode_unhashed(inode))) {  in iget_locked()
1308 			iput(inode);  in iget_locked()
1311 		return inode;  in iget_locked()
1314 	inode = alloc_inode(sb);  in iget_locked()
1315 	if (inode) {  in iget_locked()
1316 		struct inode *old;  in iget_locked()
1322 			inode->i_ino = ino;  in iget_locked()
1323 			spin_lock(&inode->i_lock);  in iget_locked()
1324 			inode->i_state = I_NEW;  in iget_locked()
1325 			hlist_add_head_rcu(&inode->i_hash, head);  in iget_locked()
1326 			spin_unlock(&inode->i_lock);  in iget_locked()
1327 			inode_sb_list_add(inode);  in iget_locked()
1333 			return inode;  in iget_locked()
1342 		destroy_inode(inode);  in iget_locked()
1345 		inode = old;  in iget_locked()
1346 		wait_on_inode(inode);  in iget_locked()
1347 		if (unlikely(inode_unhashed(inode))) {  in iget_locked()
1348 			iput(inode);  in iget_locked()
1352 	return inode;  in iget_locked()
1366 	struct inode *inode;  in test_inode_iunique()  local
1368 	hlist_for_each_entry_rcu(inode, b, i_hash) {  in test_inode_iunique()
1369 		if (inode->i_ino == ino && inode->i_sb == sb)  in test_inode_iunique()
1414 struct inode *igrab(struct inode *inode)  in igrab()  argument
1416 	spin_lock(&inode->i_lock);  in igrab()
1417 	if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {  in igrab()
1418 		__iget(inode);  in igrab()
1419 		spin_unlock(&inode->i_lock);  in igrab()
1421 		spin_unlock(&inode->i_lock);  in igrab()
1427 		inode = NULL;  in igrab()
1429 	return inode;  in igrab()
1449 struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,  in ilookup5_nowait()
1450 		int (*test)(struct inode *, void *), void *data)  in ilookup5_nowait()  argument
1453 	struct inode *inode;  in ilookup5_nowait()  local
1456 	inode = find_inode(sb, head, test, data);  in ilookup5_nowait()
1459 	return IS_ERR(inode) ? NULL : inode;  in ilookup5_nowait()
1480 struct inode *ilookup5(struct super_block *sb, unsigned long hashval,  in ilookup5()
1481 		int (*test)(struct inode *, void *), void *data)  in ilookup5()  argument
1483 	struct inode *inode;  in ilookup5()  local
1485 	inode = ilookup5_nowait(sb, hashval, test, data);  in ilookup5()
1486 	if (inode) {  in ilookup5()
1487 		wait_on_inode(inode);  in ilookup5()
1488 		if (unlikely(inode_unhashed(inode))) {  in ilookup5()
1489 			iput(inode);  in ilookup5()
1493 	return inode;  in ilookup5()
1505 struct inode *ilookup(struct super_block *sb, unsigned long ino)  in ilookup()
1508 	struct inode *inode;  in ilookup()  local
1511 	inode = find_inode_fast(sb, head, ino);  in ilookup()
1514 	if (inode) {  in ilookup()
1515 		if (IS_ERR(inode))  in ilookup()
1517 		wait_on_inode(inode);  in ilookup()
1518 		if (unlikely(inode_unhashed(inode))) {  in ilookup()
1519 			iput(inode);  in ilookup()
1523 	return inode;  in ilookup()
1550 struct inode *find_inode_nowait(struct super_block *sb,  in find_inode_nowait()
1552 				int (*match)(struct inode *, unsigned long,  in find_inode_nowait()  argument
1557 	struct inode *inode, *ret_inode = NULL;  in find_inode_nowait()  local
1561 	hlist_for_each_entry(inode, head, i_hash) {  in find_inode_nowait()
1562 		if (inode->i_sb != sb)  in find_inode_nowait()
1564 		mval = match(inode, hashval, data);  in find_inode_nowait()
1568 			ret_inode = inode;  in find_inode_nowait()
1598 struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,  in find_inode_rcu()
1599 			     int (*test)(struct inode *, void *), void *data)  in find_inode_rcu()  argument
1602 	struct inode *inode;  in find_inode_rcu()  local
1607 	hlist_for_each_entry_rcu(inode, head, i_hash) {  in find_inode_rcu()
1608 		if (inode->i_sb == sb &&  in find_inode_rcu()
1609 		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) &&  in find_inode_rcu()
1610 		    test(inode, data))  in find_inode_rcu()
1611 			return inode;  in find_inode_rcu()
1636 struct inode *find_inode_by_ino_rcu(struct super_block *sb,  in find_inode_by_ino_rcu()
1640 	struct inode *inode;  in find_inode_by_ino_rcu()  local
1645 	hlist_for_each_entry_rcu(inode, head, i_hash) {  in find_inode_by_ino_rcu()
1646 		if (inode->i_ino == ino &&  in find_inode_by_ino_rcu()
1647 		    inode->i_sb == sb &&  in find_inode_by_ino_rcu()
1648 		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)))  in find_inode_by_ino_rcu()
1649 		    return inode;  in find_inode_by_ino_rcu()
1655 int insert_inode_locked(struct inode *inode)  in insert_inode_locked()  argument
1657 	struct super_block *sb = inode->i_sb;  in insert_inode_locked()
1658 	ino_t ino = inode->i_ino;  in insert_inode_locked()
1662 		struct inode *old = NULL;  in insert_inode_locked()
1677 			spin_lock(&inode->i_lock);  in insert_inode_locked()
1678 			inode->i_state |= I_NEW | I_CREATING;  in insert_inode_locked()
1679 			hlist_add_head_rcu(&inode->i_hash, head);  in insert_inode_locked()
1680 			spin_unlock(&inode->i_lock);  in insert_inode_locked()
1702 int insert_inode_locked4(struct inode *inode, unsigned long hashval,  in insert_inode_locked4()  argument
1703 		int (*test)(struct inode *, void *), void *data)  in insert_inode_locked4()  argument
1705 	struct inode *old;  in insert_inode_locked4()
1707 	inode->i_state |= I_CREATING;  in insert_inode_locked4()
1708 	old = inode_insert5(inode, hashval, test, NULL, data);  in insert_inode_locked4()
1710 	if (old != inode) {  in insert_inode_locked4()
1719 int generic_delete_inode(struct inode *inode)  in generic_delete_inode()  argument
1735 static void iput_final(struct inode *inode)  in iput_final()  argument
1737 	struct super_block *sb = inode->i_sb;  in iput_final()
1738 	const struct super_operations *op = inode->i_sb->s_op;  in iput_final()
1742 	WARN_ON(inode->i_state & I_NEW);  in iput_final()
1745 		drop = op->drop_inode(inode);  in iput_final()
1747 		drop = generic_drop_inode(inode);  in iput_final()
1750 	    !(inode->i_state & I_DONTCACHE) &&  in iput_final()
1752 		__inode_add_lru(inode, true);  in iput_final()
1753 		spin_unlock(&inode->i_lock);  in iput_final()
1757 	state = inode->i_state;  in iput_final()
1759 		WRITE_ONCE(inode->i_state, state | I_WILL_FREE);  in iput_final()
1760 		spin_unlock(&inode->i_lock);  in iput_final()
1762 		write_inode_now(inode, 1);  in iput_final()
1764 		spin_lock(&inode->i_lock);  in iput_final()
1765 		state = inode->i_state;  in iput_final()
1770 	WRITE_ONCE(inode->i_state, state | I_FREEING);  in iput_final()
1771 	if (!list_empty(&inode->i_lru))  in iput_final()
1772 		inode_lru_list_del(inode);  in iput_final()
1773 	spin_unlock(&inode->i_lock);  in iput_final()
1775 	evict(inode);  in iput_final()
1787 void iput(struct inode *inode)  in iput()  argument
1789 	if (!inode)  in iput()
1791 	BUG_ON(inode->i_state & I_CLEAR);  in iput()
1793 	if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {  in iput()
1794 		if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {  in iput()
1795 			atomic_inc(&inode->i_count);  in iput()
1796 			spin_unlock(&inode->i_lock);  in iput()
1797 			trace_writeback_lazytime_iput(inode);  in iput()
1798 			mark_inode_dirty_sync(inode);  in iput()
1801 		iput_final(inode);  in iput()
1821 int bmap(struct inode *inode, sector_t *block)  in bmap()  argument
1823 	if (!inode->i_mapping->a_ops->bmap)  in bmap()
1826 	*block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block);  in bmap()
1837 static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,  in relatime_need_update()  argument
1847 	if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)  in relatime_need_update()
1852 	ctime = inode_get_ctime(inode);  in relatime_need_update()
1853 	if (timespec64_compare(&ctime, &inode->i_atime) >= 0)  in relatime_need_update()
1860 	if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)  in relatime_need_update()
1884 int inode_update_timestamps(struct inode *inode, int flags)  in inode_update_timestamps()  argument
1890 		struct timespec64 ctime = inode_get_ctime(inode);  in inode_update_timestamps()
1892 		now = inode_set_ctime_current(inode);  in inode_update_timestamps()
1895 		if (!timespec64_equal(&now, &inode->i_mtime)) {  in inode_update_timestamps()
1896 			inode->i_mtime = now;  in inode_update_timestamps()
1899 		if (IS_I_VERSION(inode) && inode_maybe_inc_iversion(inode, updated))  in inode_update_timestamps()
1902 		now = current_time(inode);  in inode_update_timestamps()
1906 		if (!timespec64_equal(&now, &inode->i_atime)) {  in inode_update_timestamps()
1907 			inode->i_atime = now;  in inode_update_timestamps()
1927 int generic_update_time(struct inode *inode, int flags)  in generic_update_time()  argument
1929 	int updated = inode_update_timestamps(inode, flags);  in generic_update_time()
1933 		dirty_flags = inode->i_sb->s_flags & SB_LAZYTIME ? I_DIRTY_TIME : I_DIRTY_SYNC;  in generic_update_time()
1936 	__mark_inode_dirty(inode, dirty_flags);  in generic_update_time()
1945 int inode_update_time(struct inode *inode, int flags)  in inode_update_time()  argument
1947 	if (inode->i_op->update_time)  in inode_update_time()
1948 		return inode->i_op->update_time(inode, flags);  in inode_update_time()
1949 	generic_update_time(inode, flags);  in inode_update_time()
1963 bool atime_needs_update(const struct path *path, struct inode *inode)  in atime_needs_update()  argument
1968 	if (inode->i_flags & S_NOATIME)  in atime_needs_update()
1974 	if (HAS_UNMAPPED_ID(mnt_idmap(mnt), inode))  in atime_needs_update()
1977 	if (IS_NOATIME(inode))  in atime_needs_update()
1979 	if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))  in atime_needs_update()
1984 	if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))  in atime_needs_update()
1987 	now = current_time(inode);  in atime_needs_update()
1989 	if (!relatime_need_update(mnt, inode, now))  in atime_needs_update()
1992 	if (timespec64_equal(&inode->i_atime, &now))  in atime_needs_update()
2001 	struct inode *inode = d_inode(path->dentry);  in touch_atime()  local
2003 	if (!atime_needs_update(path, inode))  in touch_atime()
2006 	if (!sb_start_write_trylock(inode->i_sb))  in touch_atime()
2020 	inode_update_time(inode, S_ATIME);  in touch_atime()
2023 	sb_end_write(inode->i_sb);  in touch_atime()
2035 	struct inode *inode = d_inode(dentry);  in dentry_needs_remove_privs()  local
2039 	if (IS_NOSEC(inode))  in dentry_needs_remove_privs()
2042 	mask = setattr_should_drop_suidgid(idmap, inode);  in dentry_needs_remove_privs()
2067 	struct inode *inode = file_inode(file);  in __file_remove_privs()  local
2071 	if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))  in __file_remove_privs()
2086 		inode_has_no_xattr(inode);  in __file_remove_privs()
2105 static int inode_needs_update_time(struct inode *inode)  in inode_needs_update_time()  argument
2108 	struct timespec64 now = current_time(inode);  in inode_needs_update_time()
2112 	if (IS_NOCMTIME(inode))  in inode_needs_update_time()
2115 	if (!timespec64_equal(&inode->i_mtime, &now))  in inode_needs_update_time()
2118 	ctime = inode_get_ctime(inode);  in inode_needs_update_time()
2122 	if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))  in inode_needs_update_time()
2131 	struct inode *inode = file_inode(file);  in __file_update_time()  local
2135 		ret = inode_update_time(inode, sync_mode);  in __file_update_time()
2159 	struct inode *inode = file_inode(file);  in file_update_time()  local
2161 	ret = inode_needs_update_time(inode);  in file_update_time()
2187 	struct inode *inode = file_inode(file);  in file_modified_flags()  local
2200 	ret = inode_needs_update_time(inode);  in file_modified_flags()
2243 int inode_needs_sync(struct inode *inode)  in inode_needs_sync()  argument
2245 	if (IS_SYNC(inode))  in inode_needs_sync()
2247 	if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))  in inode_needs_sync()
2264 static void __wait_on_freeing_inode(struct inode *inode)  in __wait_on_freeing_inode()  argument
2267 	DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);  in __wait_on_freeing_inode()
2268 	wq = bit_waitqueue(&inode->i_state, __I_NEW);  in __wait_on_freeing_inode()
2270 	spin_unlock(&inode->i_lock);  in __wait_on_freeing_inode()
2314 					 sizeof(struct inode),  in inode_init()
2336 void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)  in init_special_inode()  argument
2338 	inode->i_mode = mode;  in init_special_inode()
2340 		inode->i_fop = &def_chr_fops;  in init_special_inode()
2341 		inode->i_rdev = rdev;  in init_special_inode()
2344 			inode->i_fop = &def_blk_fops;  in init_special_inode()
2345 		inode->i_rdev = rdev;  in init_special_inode()
2347 		inode->i_fop = &pipefifo_fops;  in init_special_inode()
2352 				  " inode %s:%lu\n", mode, inode->i_sb->s_id,  in init_special_inode()
2353 				  inode->i_ino);  in init_special_inode()
2370 void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,  in inode_init_owner()  argument
2371 		      const struct inode *dir, umode_t mode)  in inode_init_owner()
2373 	inode_fsuid_set(inode, idmap);  in inode_init_owner()
2375 		inode->i_gid = dir->i_gid;  in inode_init_owner()
2381 		inode_fsgid_set(inode, idmap);  in inode_init_owner()
2382 	inode->i_mode = mode;  in inode_init_owner()
2401 			    const struct inode *inode)  in inode_owner_or_capable()  argument
2406 	vfsuid = i_uid_into_vfsuid(idmap, inode);  in inode_owner_or_capable()
2420 static void __inode_dio_wait(struct inode *inode)  in __inode_dio_wait()  argument
2422 	wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);  in __inode_dio_wait()
2423 	DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);  in __inode_dio_wait()
2427 		if (atomic_read(&inode->i_dio_count))  in __inode_dio_wait()
2429 	} while (atomic_read(&inode->i_dio_count));  in __inode_dio_wait()
2443 void inode_dio_wait(struct inode *inode)  in inode_dio_wait()  argument
2445 	if (atomic_read(&inode->i_dio_count))  in inode_dio_wait()
2446 		__inode_dio_wait(inode);  in inode_dio_wait()
2466 void inode_set_flags(struct inode *inode, unsigned int flags,  in inode_set_flags()  argument
2470 	set_mask_bits(&inode->i_flags, mask, flags);  in inode_set_flags()
2474 void inode_nohighmem(struct inode *inode)  in inode_nohighmem()  argument
2476 	mapping_set_gfp_mask(inode->i_mapping, GFP_USER);  in inode_nohighmem()
2489 struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)  in timestamp_truncate()  argument
2491 	struct super_block *sb = inode->i_sb;  in timestamp_truncate()
2521 struct timespec64 current_time(struct inode *inode)  in current_time()  argument
2526 	return timestamp_truncate(now, inode);  in current_time()
2537 struct timespec64 inode_set_ctime_current(struct inode *inode)  in inode_set_ctime_current()  argument
2539 	struct timespec64 now = current_time(inode);  in inode_set_ctime_current()
2541 	inode_set_ctime(inode, now.tv_sec, now.tv_nsec);  in inode_set_ctime_current()
2559 			 const struct inode *inode, vfsgid_t vfsgid)  in in_group_or_capable()  argument
2563 	if (capable_wrt_inode_uidgid(idmap, inode, CAP_FSETID))  in in_group_or_capable()
2583 			const struct inode *dir, umode_t mode)  in mode_strip_sgid()