Lines Matching refs:dentry
107 static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent, in in_lookup_hash()
217 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount) in dentry_cmp() argument
235 const unsigned char *cs = READ_ONCE(dentry->d_name.name); in dentry_cmp()
248 static inline struct external_name *external_name(struct dentry *dentry) in external_name() argument
250 return container_of(dentry->d_name.name, struct external_name, name[0]); in external_name()
255 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); in __d_free() local
257 kmem_cache_free(dentry_cache, dentry); in __d_free()
274 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); in __d_free_external() local
276 __d_free_external_name(&external_name(dentry)->u.head); in __d_free_external()
278 kmem_cache_free(dentry_cache, dentry); in __d_free_external()
281 static inline int dname_external(const struct dentry *dentry) in dname_external() argument
283 return dentry->d_name.name != dentry->d_iname; in dname_external()
286 void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry) in take_dentry_name_snapshot() argument
288 spin_lock(&dentry->d_lock); in take_dentry_name_snapshot()
289 if (unlikely(dname_external(dentry))) { in take_dentry_name_snapshot()
290 struct external_name *p = external_name(dentry); in take_dentry_name_snapshot()
292 spin_unlock(&dentry->d_lock); in take_dentry_name_snapshot()
295 memcpy(name->inline_name, dentry->d_iname, in take_dentry_name_snapshot()
296 dentry->d_name.len + 1); in take_dentry_name_snapshot()
297 spin_unlock(&dentry->d_lock); in take_dentry_name_snapshot()
314 static inline void __d_set_inode_and_type(struct dentry *dentry, in __d_set_inode_and_type() argument
320 dentry->d_inode = inode; in __d_set_inode_and_type()
321 flags = READ_ONCE(dentry->d_flags); in __d_set_inode_and_type()
324 WRITE_ONCE(dentry->d_flags, flags); in __d_set_inode_and_type()
327 static inline void __d_clear_type_and_inode(struct dentry *dentry) in __d_clear_type_and_inode() argument
329 unsigned flags = READ_ONCE(dentry->d_flags); in __d_clear_type_and_inode()
332 WRITE_ONCE(dentry->d_flags, flags); in __d_clear_type_and_inode()
333 dentry->d_inode = NULL; in __d_clear_type_and_inode()
336 static void dentry_free(struct dentry *dentry) in dentry_free() argument
338 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias)); in dentry_free()
339 if (unlikely(dname_external(dentry))) { in dentry_free()
340 struct external_name *p = external_name(dentry); in dentry_free()
342 call_rcu(&dentry->d_u.d_rcu, __d_free_external); in dentry_free()
347 if (!(dentry->d_flags & DCACHE_RCUACCESS)) in dentry_free()
348 __d_free(&dentry->d_u.d_rcu); in dentry_free()
350 call_rcu(&dentry->d_u.d_rcu, __d_free); in dentry_free()
357 static void dentry_unlink_inode(struct dentry * dentry) in dentry_unlink_inode() argument
358 __releases(dentry->d_lock) in dentry_unlink_inode()
359 __releases(dentry->d_inode->i_lock) in dentry_unlink_inode()
361 struct inode *inode = dentry->d_inode; in dentry_unlink_inode()
363 raw_write_seqcount_begin(&dentry->d_seq); in dentry_unlink_inode()
364 __d_clear_type_and_inode(dentry); in dentry_unlink_inode()
365 hlist_del_init(&dentry->d_u.d_alias); in dentry_unlink_inode()
366 raw_write_seqcount_end(&dentry->d_seq); in dentry_unlink_inode()
367 spin_unlock(&dentry->d_lock); in dentry_unlink_inode()
371 if (dentry->d_op && dentry->d_op->d_iput) in dentry_unlink_inode()
372 dentry->d_op->d_iput(dentry, inode); in dentry_unlink_inode()
391 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_… argument
392 static void d_lru_add(struct dentry *dentry) in d_lru_add() argument
394 D_FLAG_VERIFY(dentry, 0); in d_lru_add()
395 dentry->d_flags |= DCACHE_LRU_LIST; in d_lru_add()
397 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); in d_lru_add()
400 static void d_lru_del(struct dentry *dentry) in d_lru_del() argument
402 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_del()
403 dentry->d_flags &= ~DCACHE_LRU_LIST; in d_lru_del()
405 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); in d_lru_del()
408 static void d_shrink_del(struct dentry *dentry) in d_shrink_del() argument
410 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); in d_shrink_del()
411 list_del_init(&dentry->d_lru); in d_shrink_del()
412 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); in d_shrink_del()
416 static void d_shrink_add(struct dentry *dentry, struct list_head *list) in d_shrink_add() argument
418 D_FLAG_VERIFY(dentry, 0); in d_shrink_add()
419 list_add(&dentry->d_lru, list); in d_shrink_add()
420 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST; in d_shrink_add()
430 static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry) in d_lru_isolate() argument
432 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_isolate()
433 dentry->d_flags &= ~DCACHE_LRU_LIST; in d_lru_isolate()
435 list_lru_isolate(lru, &dentry->d_lru); in d_lru_isolate()
438 static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry, in d_lru_shrink_move() argument
441 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_shrink_move()
442 dentry->d_flags |= DCACHE_SHRINK_LIST; in d_lru_shrink_move()
443 list_lru_isolate_move(lru, &dentry->d_lru, list); in d_lru_shrink_move()
463 static void ___d_drop(struct dentry *dentry) in ___d_drop() argument
471 if (unlikely(IS_ROOT(dentry))) in ___d_drop()
472 b = &dentry->d_sb->s_roots; in ___d_drop()
474 b = d_hash(dentry->d_name.hash); in ___d_drop()
477 __hlist_bl_del(&dentry->d_hash); in ___d_drop()
481 void __d_drop(struct dentry *dentry) in __d_drop() argument
483 if (!d_unhashed(dentry)) { in __d_drop()
484 ___d_drop(dentry); in __d_drop()
485 dentry->d_hash.pprev = NULL; in __d_drop()
486 write_seqcount_invalidate(&dentry->d_seq); in __d_drop()
491 void d_drop(struct dentry *dentry) in d_drop() argument
493 spin_lock(&dentry->d_lock); in d_drop()
494 __d_drop(dentry); in d_drop()
495 spin_unlock(&dentry->d_lock); in d_drop()
499 static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent) in dentry_unlist() argument
501 struct dentry *next; in dentry_unlist()
506 dentry->d_flags |= DCACHE_DENTRY_KILLED; in dentry_unlist()
507 if (unlikely(list_empty(&dentry->d_child))) in dentry_unlist()
509 __list_del_entry(&dentry->d_child); in dentry_unlist()
529 while (dentry->d_child.next != &parent->d_subdirs) { in dentry_unlist()
530 next = list_entry(dentry->d_child.next, struct dentry, d_child); in dentry_unlist()
533 dentry->d_child.next = next->d_child.next; in dentry_unlist()
537 static void __dentry_kill(struct dentry *dentry) in __dentry_kill() argument
539 struct dentry *parent = NULL; in __dentry_kill()
541 if (!IS_ROOT(dentry)) in __dentry_kill()
542 parent = dentry->d_parent; in __dentry_kill()
547 lockref_mark_dead(&dentry->d_lockref); in __dentry_kill()
553 if (dentry->d_flags & DCACHE_OP_PRUNE) in __dentry_kill()
554 dentry->d_op->d_prune(dentry); in __dentry_kill()
556 if (dentry->d_flags & DCACHE_LRU_LIST) { in __dentry_kill()
557 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) in __dentry_kill()
558 d_lru_del(dentry); in __dentry_kill()
561 __d_drop(dentry); in __dentry_kill()
562 dentry_unlist(dentry, parent); in __dentry_kill()
565 if (dentry->d_inode) in __dentry_kill()
566 dentry_unlink_inode(dentry); in __dentry_kill()
568 spin_unlock(&dentry->d_lock); in __dentry_kill()
570 if (dentry->d_op && dentry->d_op->d_release) in __dentry_kill()
571 dentry->d_op->d_release(dentry); in __dentry_kill()
573 spin_lock(&dentry->d_lock); in __dentry_kill()
574 if (dentry->d_flags & DCACHE_SHRINK_LIST) { in __dentry_kill()
575 dentry->d_flags |= DCACHE_MAY_FREE; in __dentry_kill()
578 spin_unlock(&dentry->d_lock); in __dentry_kill()
580 dentry_free(dentry); in __dentry_kill()
584 static struct dentry *__lock_parent(struct dentry *dentry) in __lock_parent() argument
586 struct dentry *parent; in __lock_parent()
588 spin_unlock(&dentry->d_lock); in __lock_parent()
590 parent = READ_ONCE(dentry->d_parent); in __lock_parent()
600 if (unlikely(parent != dentry->d_parent)) { in __lock_parent()
605 if (parent != dentry) in __lock_parent()
606 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in __lock_parent()
612 static inline struct dentry *lock_parent(struct dentry *dentry) in lock_parent() argument
614 struct dentry *parent = dentry->d_parent; in lock_parent()
615 if (IS_ROOT(dentry)) in lock_parent()
619 return __lock_parent(dentry); in lock_parent()
622 static inline bool retain_dentry(struct dentry *dentry) in retain_dentry() argument
624 WARN_ON(d_in_lookup(dentry)); in retain_dentry()
627 if (unlikely(d_unhashed(dentry))) in retain_dentry()
630 if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) in retain_dentry()
633 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) { in retain_dentry()
634 if (dentry->d_op->d_delete(dentry)) in retain_dentry()
638 dentry->d_lockref.count--; in retain_dentry()
639 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) in retain_dentry()
640 d_lru_add(dentry); in retain_dentry()
641 else if (unlikely(!(dentry->d_flags & DCACHE_REFERENCED))) in retain_dentry()
642 dentry->d_flags |= DCACHE_REFERENCED; in retain_dentry()
651 static struct dentry *dentry_kill(struct dentry *dentry) in dentry_kill() argument
652 __releases(dentry->d_lock) in dentry_kill()
654 struct inode *inode = dentry->d_inode; in dentry_kill()
655 struct dentry *parent = NULL; in dentry_kill()
660 if (!IS_ROOT(dentry)) { in dentry_kill()
661 parent = dentry->d_parent; in dentry_kill()
663 parent = __lock_parent(dentry); in dentry_kill()
664 if (likely(inode || !dentry->d_inode)) in dentry_kill()
669 inode = dentry->d_inode; in dentry_kill()
673 __dentry_kill(dentry); in dentry_kill()
677 spin_unlock(&dentry->d_lock); in dentry_kill()
679 spin_lock(&dentry->d_lock); in dentry_kill()
680 parent = lock_parent(dentry); in dentry_kill()
682 if (unlikely(dentry->d_lockref.count != 1)) { in dentry_kill()
683 dentry->d_lockref.count--; in dentry_kill()
684 } else if (likely(!retain_dentry(dentry))) { in dentry_kill()
685 __dentry_kill(dentry); in dentry_kill()
693 spin_unlock(&dentry->d_lock); in dentry_kill()
705 static inline bool fast_dput(struct dentry *dentry) in fast_dput() argument
714 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) in fast_dput()
715 return lockref_put_or_lock(&dentry->d_lockref); in fast_dput()
721 ret = lockref_put_return(&dentry->d_lockref); in fast_dput()
729 spin_lock(&dentry->d_lock); in fast_dput()
730 if (dentry->d_lockref.count > 1) { in fast_dput()
731 dentry->d_lockref.count--; in fast_dput()
732 spin_unlock(&dentry->d_lock); in fast_dput()
766 d_flags = READ_ONCE(dentry->d_flags); in fast_dput()
770 if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry)) in fast_dput()
778 spin_lock(&dentry->d_lock); in fast_dput()
786 if (dentry->d_lockref.count) { in fast_dput()
787 spin_unlock(&dentry->d_lock); in fast_dput()
796 dentry->d_lockref.count = 1; in fast_dput()
827 void dput(struct dentry *dentry) in dput() argument
829 while (dentry) { in dput()
833 if (likely(fast_dput(dentry))) { in dput()
841 if (likely(retain_dentry(dentry))) { in dput()
842 spin_unlock(&dentry->d_lock); in dput()
846 dentry = dentry_kill(dentry); in dput()
853 static inline void __dget_dlock(struct dentry *dentry) in __dget_dlock() argument
855 dentry->d_lockref.count++; in __dget_dlock()
858 static inline void __dget(struct dentry *dentry) in __dget() argument
860 lockref_get(&dentry->d_lockref); in __dget()
863 struct dentry *dget_parent(struct dentry *dentry) in dget_parent() argument
866 struct dentry *ret; in dget_parent()
873 ret = READ_ONCE(dentry->d_parent); in dget_parent()
877 if (likely(ret == READ_ONCE(dentry->d_parent))) in dget_parent()
888 ret = dentry->d_parent; in dget_parent()
890 if (unlikely(ret != dentry->d_parent)) { in dget_parent()
903 static struct dentry * __d_find_any_alias(struct inode *inode) in __d_find_any_alias()
905 struct dentry *alias; in __d_find_any_alias()
909 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias); in __d_find_any_alias()
921 struct dentry *d_find_any_alias(struct inode *inode) in d_find_any_alias()
923 struct dentry *de; in d_find_any_alias()
946 static struct dentry *__d_find_alias(struct inode *inode) in __d_find_alias()
948 struct dentry *alias; in __d_find_alias()
965 struct dentry *d_find_alias(struct inode *inode) in d_find_alias()
967 struct dentry *de = NULL; in d_find_alias()
984 struct dentry *dentry; in d_prune_aliases() local
987 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { in d_prune_aliases()
988 spin_lock(&dentry->d_lock); in d_prune_aliases()
989 if (!dentry->d_lockref.count) { in d_prune_aliases()
990 struct dentry *parent = lock_parent(dentry); in d_prune_aliases()
991 if (likely(!dentry->d_lockref.count)) { in d_prune_aliases()
992 __dentry_kill(dentry); in d_prune_aliases()
999 spin_unlock(&dentry->d_lock); in d_prune_aliases()
1016 static bool shrink_lock_dentry(struct dentry *dentry) in shrink_lock_dentry() argument
1019 struct dentry *parent; in shrink_lock_dentry()
1021 if (dentry->d_lockref.count) in shrink_lock_dentry()
1024 inode = dentry->d_inode; in shrink_lock_dentry()
1026 spin_unlock(&dentry->d_lock); in shrink_lock_dentry()
1028 spin_lock(&dentry->d_lock); in shrink_lock_dentry()
1029 if (unlikely(dentry->d_lockref.count)) in shrink_lock_dentry()
1032 if (unlikely(inode != dentry->d_inode)) in shrink_lock_dentry()
1036 parent = dentry->d_parent; in shrink_lock_dentry()
1037 if (IS_ROOT(dentry) || likely(spin_trylock(&parent->d_lock))) in shrink_lock_dentry()
1040 spin_unlock(&dentry->d_lock); in shrink_lock_dentry()
1042 if (unlikely(parent != dentry->d_parent)) { in shrink_lock_dentry()
1044 spin_lock(&dentry->d_lock); in shrink_lock_dentry()
1047 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in shrink_lock_dentry()
1048 if (likely(!dentry->d_lockref.count)) in shrink_lock_dentry()
1060 struct dentry *dentry, *parent; in shrink_dentry_list() local
1062 dentry = list_entry(list->prev, struct dentry, d_lru); in shrink_dentry_list()
1063 spin_lock(&dentry->d_lock); in shrink_dentry_list()
1065 if (!shrink_lock_dentry(dentry)) { in shrink_dentry_list()
1068 d_shrink_del(dentry); in shrink_dentry_list()
1069 if (dentry->d_lockref.count < 0) in shrink_dentry_list()
1070 can_free = dentry->d_flags & DCACHE_MAY_FREE; in shrink_dentry_list()
1071 spin_unlock(&dentry->d_lock); in shrink_dentry_list()
1073 dentry_free(dentry); in shrink_dentry_list()
1077 d_shrink_del(dentry); in shrink_dentry_list()
1078 parent = dentry->d_parent; in shrink_dentry_list()
1079 __dentry_kill(dentry); in shrink_dentry_list()
1080 if (parent == dentry) in shrink_dentry_list()
1088 dentry = parent; in shrink_dentry_list()
1089 while (dentry && !lockref_put_or_lock(&dentry->d_lockref)) in shrink_dentry_list()
1090 dentry = dentry_kill(dentry); in shrink_dentry_list()
1098 struct dentry *dentry = container_of(item, struct dentry, d_lru); in dentry_lru_isolate() local
1106 if (!spin_trylock(&dentry->d_lock)) in dentry_lru_isolate()
1114 if (dentry->d_lockref.count) { in dentry_lru_isolate()
1115 d_lru_isolate(lru, dentry); in dentry_lru_isolate()
1116 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1120 if (dentry->d_flags & DCACHE_REFERENCED) { in dentry_lru_isolate()
1121 dentry->d_flags &= ~DCACHE_REFERENCED; in dentry_lru_isolate()
1122 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1146 d_lru_shrink_move(lru, dentry, freeable); in dentry_lru_isolate()
1147 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1179 struct dentry *dentry = container_of(item, struct dentry, d_lru); in dentry_lru_isolate_shrink() local
1186 if (!spin_trylock(&dentry->d_lock)) in dentry_lru_isolate_shrink()
1189 d_lru_shrink_move(lru, dentry, freeable); in dentry_lru_isolate_shrink()
1190 spin_unlock(&dentry->d_lock); in dentry_lru_isolate_shrink()
1241 static void d_walk(struct dentry *parent, void *data, in d_walk() argument
1242 enum d_walk_ret (*enter)(void *, struct dentry *)) in d_walk()
1244 struct dentry *this_parent; in d_walk()
1271 struct dentry *dentry = list_entry(tmp, struct dentry, d_child); in d_walk() local
1274 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR)) in d_walk()
1277 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in d_walk()
1279 ret = enter(data, dentry); in d_walk()
1284 spin_unlock(&dentry->d_lock); in d_walk()
1290 spin_unlock(&dentry->d_lock); in d_walk()
1294 if (!list_empty(&dentry->d_subdirs)) { in d_walk()
1296 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_); in d_walk()
1297 this_parent = dentry; in d_walk()
1301 spin_unlock(&dentry->d_lock); in d_walk()
1309 struct dentry *child = this_parent; in d_walk()
1323 child = list_entry(next, struct dentry, d_child); in d_walk()
1352 static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry) in path_check_mount() argument
1355 struct path path = { .mnt = info->mnt, .dentry = dentry }; in path_check_mount()
1357 if (likely(!d_mountpoint(dentry))) in path_check_mount()
1379 d_walk(parent->dentry, &data, path_check_mount); in path_has_submounts()
1394 int d_set_mounted(struct dentry *dentry) in d_set_mounted() argument
1396 struct dentry *p; in d_set_mounted()
1399 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) { in d_set_mounted()
1408 spin_lock(&dentry->d_lock); in d_set_mounted()
1409 if (!d_unlinked(dentry)) { in d_set_mounted()
1411 if (!d_mountpoint(dentry)) { in d_set_mounted()
1412 dentry->d_flags |= DCACHE_MOUNTED; in d_set_mounted()
1416 spin_unlock(&dentry->d_lock); in d_set_mounted()
1438 struct dentry *start;
1443 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) in select_collect() argument
1448 if (data->start == dentry) in select_collect()
1451 if (dentry->d_flags & DCACHE_SHRINK_LIST) { in select_collect()
1454 if (dentry->d_flags & DCACHE_LRU_LIST) in select_collect()
1455 d_lru_del(dentry); in select_collect()
1456 if (!dentry->d_lockref.count) { in select_collect()
1457 d_shrink_add(dentry, &data->dispose); in select_collect()
1478 void shrink_dcache_parent(struct dentry *parent) in shrink_dcache_parent()
1501 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry) in umount_check() argument
1504 if (!list_empty(&dentry->d_subdirs)) in umount_check()
1508 if (dentry == _data && dentry->d_lockref.count == 1) in umount_check()
1513 dentry, in umount_check()
1514 dentry->d_inode ? in umount_check()
1515 dentry->d_inode->i_ino : 0UL, in umount_check()
1516 dentry, in umount_check()
1517 dentry->d_lockref.count, in umount_check()
1518 dentry->d_sb->s_type->name, in umount_check()
1519 dentry->d_sb->s_id); in umount_check()
1524 static void do_one_tree(struct dentry *dentry) in do_one_tree() argument
1526 shrink_dcache_parent(dentry); in do_one_tree()
1527 d_walk(dentry, dentry, umount_check); in do_one_tree()
1528 d_drop(dentry); in do_one_tree()
1529 dput(dentry); in do_one_tree()
1537 struct dentry *dentry; in shrink_dcache_for_umount() local
1541 dentry = sb->s_root; in shrink_dcache_for_umount()
1543 do_one_tree(dentry); in shrink_dcache_for_umount()
1546 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash)); in shrink_dcache_for_umount()
1547 do_one_tree(dentry); in shrink_dcache_for_umount()
1551 static enum d_walk_ret find_submount(void *_data, struct dentry *dentry) in find_submount() argument
1553 struct dentry **victim = _data; in find_submount()
1554 if (d_mountpoint(dentry)) { in find_submount()
1555 __dget_dlock(dentry); in find_submount()
1556 *victim = dentry; in find_submount()
1566 void d_invalidate(struct dentry *dentry) in d_invalidate() argument
1569 spin_lock(&dentry->d_lock); in d_invalidate()
1570 if (d_unhashed(dentry)) { in d_invalidate()
1571 spin_unlock(&dentry->d_lock); in d_invalidate()
1574 __d_drop(dentry); in d_invalidate()
1575 spin_unlock(&dentry->d_lock); in d_invalidate()
1578 if (!dentry->d_inode) in d_invalidate()
1581 shrink_dcache_parent(dentry); in d_invalidate()
1583 struct dentry *victim = NULL; in d_invalidate()
1584 d_walk(dentry, &victim, find_submount); in d_invalidate()
1587 shrink_dcache_parent(dentry); in d_invalidate()
1607 struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) in __d_alloc()
1610 struct dentry *dentry; in __d_alloc() local
1614 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL); in __d_alloc()
1615 if (!dentry) in __d_alloc()
1624 dentry->d_iname[DNAME_INLINE_LEN-1] = 0; in __d_alloc()
1627 dname = dentry->d_iname; in __d_alloc()
1633 kmem_cache_free(dentry_cache, dentry); in __d_alloc()
1639 dname = dentry->d_iname; in __d_alloc()
1642 dentry->d_name.len = name->len; in __d_alloc()
1643 dentry->d_name.hash = name->hash; in __d_alloc()
1648 smp_store_release(&dentry->d_name.name, dname); /* ^^^ */ in __d_alloc()
1650 dentry->d_lockref.count = 1; in __d_alloc()
1651 dentry->d_flags = 0; in __d_alloc()
1652 spin_lock_init(&dentry->d_lock); in __d_alloc()
1653 seqcount_init(&dentry->d_seq); in __d_alloc()
1654 dentry->d_inode = NULL; in __d_alloc()
1655 dentry->d_parent = dentry; in __d_alloc()
1656 dentry->d_sb = sb; in __d_alloc()
1657 dentry->d_op = NULL; in __d_alloc()
1658 dentry->d_fsdata = NULL; in __d_alloc()
1659 INIT_HLIST_BL_NODE(&dentry->d_hash); in __d_alloc()
1660 INIT_LIST_HEAD(&dentry->d_lru); in __d_alloc()
1661 INIT_LIST_HEAD(&dentry->d_subdirs); in __d_alloc()
1662 INIT_HLIST_NODE(&dentry->d_u.d_alias); in __d_alloc()
1663 INIT_LIST_HEAD(&dentry->d_child); in __d_alloc()
1664 d_set_d_op(dentry, dentry->d_sb->s_d_op); in __d_alloc()
1666 if (dentry->d_op && dentry->d_op->d_init) { in __d_alloc()
1667 err = dentry->d_op->d_init(dentry); in __d_alloc()
1669 if (dname_external(dentry)) in __d_alloc()
1670 kfree(external_name(dentry)); in __d_alloc()
1671 kmem_cache_free(dentry_cache, dentry); in __d_alloc()
1684 return dentry; in __d_alloc()
1696 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) in d_alloc()
1698 struct dentry *dentry = __d_alloc(parent->d_sb, name); in d_alloc() local
1699 if (!dentry) in d_alloc()
1701 dentry->d_flags |= DCACHE_RCUACCESS; in d_alloc()
1708 dentry->d_parent = parent; in d_alloc()
1709 list_add(&dentry->d_child, &parent->d_subdirs); in d_alloc()
1712 return dentry; in d_alloc()
1716 struct dentry *d_alloc_anon(struct super_block *sb) in d_alloc_anon()
1722 struct dentry *d_alloc_cursor(struct dentry * parent) in d_alloc_cursor()
1724 struct dentry *dentry = d_alloc_anon(parent->d_sb); in d_alloc_cursor() local
1725 if (dentry) { in d_alloc_cursor()
1726 dentry->d_flags |= DCACHE_RCUACCESS | DCACHE_DENTRY_CURSOR; in d_alloc_cursor()
1727 dentry->d_parent = dget(parent); in d_alloc_cursor()
1729 return dentry; in d_alloc_cursor()
1740 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name) in d_alloc_pseudo()
1746 struct dentry *d_alloc_name(struct dentry *parent, const char *name) in d_alloc_name()
1756 void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op) in d_set_d_op() argument
1758 WARN_ON_ONCE(dentry->d_op); in d_set_d_op()
1759 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH | in d_set_d_op()
1765 dentry->d_op = op; in d_set_d_op()
1769 dentry->d_flags |= DCACHE_OP_HASH; in d_set_d_op()
1771 dentry->d_flags |= DCACHE_OP_COMPARE; in d_set_d_op()
1773 dentry->d_flags |= DCACHE_OP_REVALIDATE; in d_set_d_op()
1775 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE; in d_set_d_op()
1777 dentry->d_flags |= DCACHE_OP_DELETE; in d_set_d_op()
1779 dentry->d_flags |= DCACHE_OP_PRUNE; in d_set_d_op()
1781 dentry->d_flags |= DCACHE_OP_REAL; in d_set_d_op()
1794 void d_set_fallthru(struct dentry *dentry) in d_set_fallthru() argument
1796 spin_lock(&dentry->d_lock); in d_set_fallthru()
1797 dentry->d_flags |= DCACHE_FALLTHRU; in d_set_fallthru()
1798 spin_unlock(&dentry->d_lock); in d_set_fallthru()
1837 static void __d_instantiate(struct dentry *dentry, struct inode *inode) in __d_instantiate() argument
1840 WARN_ON(d_in_lookup(dentry)); in __d_instantiate()
1842 spin_lock(&dentry->d_lock); in __d_instantiate()
1843 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); in __d_instantiate()
1844 raw_write_seqcount_begin(&dentry->d_seq); in __d_instantiate()
1845 __d_set_inode_and_type(dentry, inode, add_flags); in __d_instantiate()
1846 raw_write_seqcount_end(&dentry->d_seq); in __d_instantiate()
1847 fsnotify_update_flags(dentry); in __d_instantiate()
1848 spin_unlock(&dentry->d_lock); in __d_instantiate()
1866 void d_instantiate(struct dentry *entry, struct inode * inode) in d_instantiate()
1884 void d_instantiate_new(struct dentry *entry, struct inode *inode) in d_instantiate_new()
1900 struct dentry *d_make_root(struct inode *root_inode) in d_make_root()
1902 struct dentry *res = NULL; in d_make_root()
1917 static struct dentry *__d_instantiate_anon(struct dentry *dentry, in __d_instantiate_anon() argument
1921 struct dentry *res; in __d_instantiate_anon()
1924 security_d_instantiate(dentry, inode); in __d_instantiate_anon()
1929 dput(dentry); in __d_instantiate_anon()
1939 spin_lock(&dentry->d_lock); in __d_instantiate_anon()
1940 __d_set_inode_and_type(dentry, inode, add_flags); in __d_instantiate_anon()
1941 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); in __d_instantiate_anon()
1943 hlist_bl_lock(&dentry->d_sb->s_roots); in __d_instantiate_anon()
1944 hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots); in __d_instantiate_anon()
1945 hlist_bl_unlock(&dentry->d_sb->s_roots); in __d_instantiate_anon()
1947 spin_unlock(&dentry->d_lock); in __d_instantiate_anon()
1950 return dentry; in __d_instantiate_anon()
1957 struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode) in d_instantiate_anon() argument
1959 return __d_instantiate_anon(dentry, inode, true); in d_instantiate_anon()
1963 static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected) in __d_obtain_alias()
1965 struct dentry *tmp; in __d_obtain_alias()
1966 struct dentry *res; in __d_obtain_alias()
2008 struct dentry *d_obtain_alias(struct inode *inode) in d_obtain_alias()
2029 struct dentry *d_obtain_root(struct inode *inode) in d_obtain_root()
2051 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, in d_add_ci() argument
2054 struct dentry *found, *res; in d_add_ci()
2060 found = d_hash_and_lookup(dentry->d_parent, name); in d_add_ci()
2065 if (d_in_lookup(dentry)) { in d_add_ci()
2066 found = d_alloc_parallel(dentry->d_parent, name, in d_add_ci()
2067 dentry->d_wait); in d_add_ci()
2073 found = d_alloc(dentry->d_parent, name); in d_add_ci()
2089 static inline bool d_same_name(const struct dentry *dentry, in d_same_name() argument
2090 const struct dentry *parent, in d_same_name()
2094 if (dentry->d_name.len != name->len) in d_same_name()
2096 return dentry_cmp(dentry, name->name, name->len) == 0; in d_same_name()
2098 return parent->d_op->d_compare(dentry, in d_same_name()
2099 dentry->d_name.len, dentry->d_name.name, in d_same_name()
2132 struct dentry *__d_lookup_rcu(const struct dentry *parent, in __d_lookup_rcu()
2140 struct dentry *dentry; in __d_lookup_rcu() local
2162 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup_rcu()
2183 seq = raw_seqcount_begin(&dentry->d_seq); in __d_lookup_rcu()
2184 if (dentry->d_parent != parent) in __d_lookup_rcu()
2186 if (d_unhashed(dentry)) in __d_lookup_rcu()
2192 if (dentry->d_name.hash != hashlen_hash(hashlen)) in __d_lookup_rcu()
2194 tlen = dentry->d_name.len; in __d_lookup_rcu()
2195 tname = dentry->d_name.name; in __d_lookup_rcu()
2197 if (read_seqcount_retry(&dentry->d_seq, seq)) { in __d_lookup_rcu()
2201 if (parent->d_op->d_compare(dentry, in __d_lookup_rcu()
2205 if (dentry->d_name.hash_len != hashlen) in __d_lookup_rcu()
2207 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) in __d_lookup_rcu()
2211 return dentry; in __d_lookup_rcu()
2227 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name) in d_lookup()
2229 struct dentry *dentry; in d_lookup() local
2234 dentry = __d_lookup(parent, name); in d_lookup()
2235 if (dentry) in d_lookup()
2238 return dentry; in d_lookup()
2257 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) in __d_lookup()
2262 struct dentry *found = NULL; in __d_lookup()
2263 struct dentry *dentry; in __d_lookup() local
2287 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup()
2289 if (dentry->d_name.hash != hash) in __d_lookup()
2292 spin_lock(&dentry->d_lock); in __d_lookup()
2293 if (dentry->d_parent != parent) in __d_lookup()
2295 if (d_unhashed(dentry)) in __d_lookup()
2298 if (!d_same_name(dentry, parent, name)) in __d_lookup()
2301 dentry->d_lockref.count++; in __d_lookup()
2302 found = dentry; in __d_lookup()
2303 spin_unlock(&dentry->d_lock); in __d_lookup()
2306 spin_unlock(&dentry->d_lock); in __d_lookup()
2320 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name) in d_hash_and_lookup()
2358 void d_delete(struct dentry * dentry) in d_delete() argument
2360 struct inode *inode = dentry->d_inode; in d_delete()
2361 int isdir = d_is_dir(dentry); in d_delete()
2364 spin_lock(&dentry->d_lock); in d_delete()
2368 if (dentry->d_lockref.count == 1) { in d_delete()
2369 dentry->d_flags &= ~DCACHE_CANT_MOUNT; in d_delete()
2370 dentry_unlink_inode(dentry); in d_delete()
2372 __d_drop(dentry); in d_delete()
2373 spin_unlock(&dentry->d_lock); in d_delete()
2376 fsnotify_nameremove(dentry, isdir); in d_delete()
2380 static void __d_rehash(struct dentry *entry) in __d_rehash()
2396 void d_rehash(struct dentry * entry) in d_rehash()
2420 static void d_wait_lookup(struct dentry *dentry) in d_wait_lookup() argument
2422 if (d_in_lookup(dentry)) { in d_wait_lookup()
2424 add_wait_queue(dentry->d_wait, &wait); in d_wait_lookup()
2427 spin_unlock(&dentry->d_lock); in d_wait_lookup()
2429 spin_lock(&dentry->d_lock); in d_wait_lookup()
2430 } while (d_in_lookup(dentry)); in d_wait_lookup()
2434 struct dentry *d_alloc_parallel(struct dentry *parent, in d_alloc_parallel()
2441 struct dentry *new = d_alloc(parent, name); in d_alloc_parallel()
2442 struct dentry *dentry; in d_alloc_parallel() local
2452 dentry = __d_lookup_rcu(parent, name, &d_seq); in d_alloc_parallel()
2453 if (unlikely(dentry)) { in d_alloc_parallel()
2454 if (!lockref_get_not_dead(&dentry->d_lockref)) { in d_alloc_parallel()
2458 if (read_seqcount_retry(&dentry->d_seq, d_seq)) { in d_alloc_parallel()
2460 dput(dentry); in d_alloc_parallel()
2465 return dentry; in d_alloc_parallel()
2490 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) { in d_alloc_parallel()
2491 if (dentry->d_name.hash != hash) in d_alloc_parallel()
2493 if (dentry->d_parent != parent) in d_alloc_parallel()
2495 if (!d_same_name(dentry, parent, name)) in d_alloc_parallel()
2499 if (!lockref_get_not_dead(&dentry->d_lockref)) { in d_alloc_parallel()
2509 spin_lock(&dentry->d_lock); in d_alloc_parallel()
2510 d_wait_lookup(dentry); in d_alloc_parallel()
2517 if (unlikely(dentry->d_name.hash != hash)) in d_alloc_parallel()
2519 if (unlikely(dentry->d_parent != parent)) in d_alloc_parallel()
2521 if (unlikely(d_unhashed(dentry))) in d_alloc_parallel()
2523 if (unlikely(!d_same_name(dentry, parent, name))) in d_alloc_parallel()
2526 spin_unlock(&dentry->d_lock); in d_alloc_parallel()
2528 return dentry; in d_alloc_parallel()
2538 spin_unlock(&dentry->d_lock); in d_alloc_parallel()
2539 dput(dentry); in d_alloc_parallel()
2544 void __d_lookup_done(struct dentry *dentry) in __d_lookup_done() argument
2546 struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent, in __d_lookup_done()
2547 dentry->d_name.hash); in __d_lookup_done()
2549 dentry->d_flags &= ~DCACHE_PAR_LOOKUP; in __d_lookup_done()
2550 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash); in __d_lookup_done()
2551 wake_up_all(dentry->d_wait); in __d_lookup_done()
2552 dentry->d_wait = NULL; in __d_lookup_done()
2554 INIT_HLIST_NODE(&dentry->d_u.d_alias); in __d_lookup_done()
2555 INIT_LIST_HEAD(&dentry->d_lru); in __d_lookup_done()
2561 static inline void __d_add(struct dentry *dentry, struct inode *inode) in __d_add() argument
2565 spin_lock(&dentry->d_lock); in __d_add()
2566 if (unlikely(d_in_lookup(dentry))) { in __d_add()
2567 dir = dentry->d_parent->d_inode; in __d_add()
2569 __d_lookup_done(dentry); in __d_add()
2573 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); in __d_add()
2574 raw_write_seqcount_begin(&dentry->d_seq); in __d_add()
2575 __d_set_inode_and_type(dentry, inode, add_flags); in __d_add()
2576 raw_write_seqcount_end(&dentry->d_seq); in __d_add()
2577 fsnotify_update_flags(dentry); in __d_add()
2579 __d_rehash(dentry); in __d_add()
2582 spin_unlock(&dentry->d_lock); in __d_add()
2596 void d_add(struct dentry *entry, struct inode *inode) in d_add()
2617 struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) in d_exact_alias()
2619 struct dentry *alias; in d_exact_alias()
2652 static void swap_names(struct dentry *dentry, struct dentry *target) in swap_names() argument
2655 if (unlikely(dname_external(dentry))) { in swap_names()
2659 swap(target->d_name.name, dentry->d_name.name); in swap_names()
2665 memcpy(target->d_iname, dentry->d_name.name, in swap_names()
2666 dentry->d_name.len + 1); in swap_names()
2667 dentry->d_name.name = target->d_name.name; in swap_names()
2671 if (unlikely(dname_external(dentry))) { in swap_names()
2676 memcpy(dentry->d_iname, target->d_name.name, in swap_names()
2678 target->d_name.name = dentry->d_name.name; in swap_names()
2679 dentry->d_name.name = dentry->d_iname; in swap_names()
2687 swap(((long *) &dentry->d_iname)[i], in swap_names()
2692 swap(dentry->d_name.hash_len, target->d_name.hash_len); in swap_names()
2695 static void copy_name(struct dentry *dentry, struct dentry *target) in copy_name() argument
2698 if (unlikely(dname_external(dentry))) in copy_name()
2699 old_name = external_name(dentry); in copy_name()
2702 dentry->d_name = target->d_name; in copy_name()
2704 memcpy(dentry->d_iname, target->d_name.name, in copy_name()
2706 dentry->d_name.name = dentry->d_iname; in copy_name()
2707 dentry->d_name.hash_len = target->d_name.hash_len; in copy_name()
2724 static void __d_move(struct dentry *dentry, struct dentry *target, in __d_move() argument
2727 struct dentry *old_parent, *p; in __d_move()
2731 WARN_ON(!dentry->d_inode); in __d_move()
2732 if (WARN_ON(dentry == target)) in __d_move()
2735 BUG_ON(d_ancestor(target, dentry)); in __d_move()
2736 old_parent = dentry->d_parent; in __d_move()
2738 if (IS_ROOT(dentry)) { in __d_move()
2746 BUG_ON(p == dentry); in __d_move()
2752 spin_lock_nested(&dentry->d_lock, 2); in __d_move()
2761 write_seqcount_begin(&dentry->d_seq); in __d_move()
2765 if (!d_unhashed(dentry)) in __d_move()
2766 ___d_drop(dentry); in __d_move()
2771 dentry->d_parent = target->d_parent; in __d_move()
2773 copy_name(dentry, target); in __d_move()
2775 dentry->d_parent->d_lockref.count++; in __d_move()
2776 if (dentry == old_parent) in __d_move()
2777 dentry->d_flags |= DCACHE_RCUACCESS; in __d_move()
2782 swap_names(dentry, target); in __d_move()
2787 list_move(&dentry->d_child, &dentry->d_parent->d_subdirs); in __d_move()
2788 __d_rehash(dentry); in __d_move()
2789 fsnotify_update_flags(dentry); in __d_move()
2792 write_seqcount_end(&dentry->d_seq); in __d_move()
2797 if (dentry->d_parent != old_parent) in __d_move()
2798 spin_unlock(&dentry->d_parent->d_lock); in __d_move()
2799 if (dentry != old_parent) in __d_move()
2802 spin_unlock(&dentry->d_lock); in __d_move()
2814 void d_move(struct dentry *dentry, struct dentry *target) in d_move() argument
2817 __d_move(dentry, target, false); in d_move()
2827 void d_exchange(struct dentry *dentry1, struct dentry *dentry2) in d_exchange()
2849 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) in d_ancestor()
2851 struct dentry *p; in d_ancestor()
2870 struct dentry *dentry, struct dentry *alias) in __d_unalias() argument
2877 if (alias->d_parent == dentry->d_parent) in __d_unalias()
2881 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex)) in __d_unalias()
2883 m1 = &dentry->d_sb->s_vfs_rename_mutex; in __d_unalias()
2888 __d_move(alias, dentry, false); in __d_unalias()
2921 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) in d_splice_alias() argument
2926 BUG_ON(!d_unhashed(dentry)); in d_splice_alias()
2931 security_d_instantiate(dentry, inode); in d_splice_alias()
2934 struct dentry *new = __d_find_any_alias(inode); in d_splice_alias()
2939 if (unlikely(d_ancestor(new, dentry))) { in d_splice_alias()
2946 dentry->d_name.name, in d_splice_alias()
2950 struct dentry *old_parent = dget(new->d_parent); in d_splice_alias()
2951 int err = __d_unalias(inode, dentry, new); in d_splice_alias()
2959 __d_move(new, dentry, false); in d_splice_alias()
2967 __d_add(dentry, inode); in d_splice_alias()
2988 bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry) in is_subdir()
3015 static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry) in d_genocide_kill() argument
3017 struct dentry *root = data; in d_genocide_kill()
3018 if (dentry != root) { in d_genocide_kill()
3019 if (d_unhashed(dentry) || !dentry->d_inode) in d_genocide_kill()
3022 if (!(dentry->d_flags & DCACHE_GENOCIDE)) { in d_genocide_kill()
3023 dentry->d_flags |= DCACHE_GENOCIDE; in d_genocide_kill()
3024 dentry->d_lockref.count--; in d_genocide_kill()
3030 void d_genocide(struct dentry *parent) in d_genocide()
3037 void d_tmpfile(struct dentry *dentry, struct inode *inode) in d_tmpfile() argument
3040 BUG_ON(dentry->d_name.name != dentry->d_iname || in d_tmpfile()
3041 !hlist_unhashed(&dentry->d_u.d_alias) || in d_tmpfile()
3042 !d_unlinked(dentry)); in d_tmpfile()
3043 spin_lock(&dentry->d_parent->d_lock); in d_tmpfile()
3044 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in d_tmpfile()
3045 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu", in d_tmpfile()
3047 spin_unlock(&dentry->d_lock); in d_tmpfile()
3048 spin_unlock(&dentry->d_parent->d_lock); in d_tmpfile()
3049 d_instantiate(dentry, inode); in d_tmpfile()
3091 dentry_cache = KMEM_CACHE_USERCOPY(dentry, in dcache_init()