27 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
30 				struct fscache_cookie *cookie);
31 static int fscache_attach_object(struct fscache_cookie *cookie,
34 static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)  in fscache_print_cookie()  argument
41 	       prefix, cookie, cookie->parent, cookie->flags,  in fscache_print_cookie()
42 	       atomic_read(&cookie->n_children),  in fscache_print_cookie()
43 	       atomic_read(&cookie->n_active));  in fscache_print_cookie()
45 	       prefix, cookie->def, cookie->netfs_data);  in fscache_print_cookie()
47 	object = READ_ONCE(cookie->backing_objects.first);  in fscache_print_cookie()
52 	pr_err("%c-key=[%u] '", prefix, cookie->key_len);  in fscache_print_cookie()
53 	k = (cookie->key_len <= sizeof(cookie->inline_key)) ?  in fscache_print_cookie()
54 		cookie->inline_key : cookie->key;  in fscache_print_cookie()
55 	for (loop = 0; loop < cookie->key_len; loop++)  in fscache_print_cookie()
60 void fscache_free_cookie(struct fscache_cookie *cookie)  in fscache_free_cookie()  argument
62 	if (cookie) {  in fscache_free_cookie()
63 		BUG_ON(!hlist_empty(&cookie->backing_objects));  in fscache_free_cookie()
64 		if (cookie->aux_len > sizeof(cookie->inline_aux))  in fscache_free_cookie()
65 			kfree(cookie->aux);  in fscache_free_cookie()
66 		if (cookie->key_len > sizeof(cookie->inline_key))  in fscache_free_cookie()
67 			kfree(cookie->key);  in fscache_free_cookie()
68 		kmem_cache_free(fscache_cookie_jar, cookie);  in fscache_free_cookie()
78 static int fscache_set_key(struct fscache_cookie *cookie,  in fscache_set_key()  argument
88 	if (index_key_len > sizeof(cookie->inline_key)) {  in fscache_set_key()
92 		cookie->key = buf;  in fscache_set_key()
94 		buf = (u32 *)cookie->inline_key;  in fscache_set_key()
102 	h = (unsigned long)cookie->parent;  in fscache_set_key()
103 	h += index_key_len + cookie->type;  in fscache_set_key()
108 	cookie->key_hash = h ^ (h >> 32);  in fscache_set_key()
147 	struct fscache_cookie *cookie;  in fscache_alloc_cookie()  local
150 	cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);  in fscache_alloc_cookie()
151 	if (!cookie)  in fscache_alloc_cookie()
154 	cookie->key_len = index_key_len;  in fscache_alloc_cookie()
155 	cookie->aux_len = aux_data_len;  in fscache_alloc_cookie()
157 	if (fscache_set_key(cookie, index_key, index_key_len) < 0)  in fscache_alloc_cookie()
160 	if (cookie->aux_len <= sizeof(cookie->inline_aux)) {  in fscache_alloc_cookie()
161 		memcpy(cookie->inline_aux, aux_data, cookie->aux_len);  in fscache_alloc_cookie()
163 		cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);  in fscache_alloc_cookie()
164 		if (!cookie->aux)  in fscache_alloc_cookie()
168 	atomic_set(&cookie->usage, 1);  in fscache_alloc_cookie()
169 	atomic_set(&cookie->n_children, 0);  in fscache_alloc_cookie()
174 	atomic_set(&cookie->n_active, 1);  in fscache_alloc_cookie()
176 	cookie->def		= def;  in fscache_alloc_cookie()
177 	cookie->parent		= parent;  in fscache_alloc_cookie()
178 	cookie->netfs_data	= netfs_data;  in fscache_alloc_cookie()
179 	cookie->flags		= (1 << FSCACHE_COOKIE_NO_DATA_YET);  in fscache_alloc_cookie()
180 	cookie->type		= def->type;  in fscache_alloc_cookie()
181 	spin_lock_init(&cookie->lock);  in fscache_alloc_cookie()
182 	spin_lock_init(&cookie->stores_lock);  in fscache_alloc_cookie()
183 	INIT_HLIST_HEAD(&cookie->backing_objects);  in fscache_alloc_cookie()
187 	INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);  in fscache_alloc_cookie()
188 	return cookie;  in fscache_alloc_cookie()
191 	fscache_free_cookie(cookie);  in fscache_alloc_cookie()
262 	struct fscache_cookie *candidate, *cookie;  in __fscache_acquire_cookie()  local
302 	cookie = fscache_hash_cookie(candidate);  in __fscache_acquire_cookie()
303 	if (!cookie) {  in __fscache_acquire_cookie()
308 	if (cookie == candidate)  in __fscache_acquire_cookie()
311 	switch (cookie->type) {  in __fscache_acquire_cookie()
323 	trace_fscache_acquire(cookie);  in __fscache_acquire_cookie()
329 		if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {  in __fscache_acquire_cookie()
330 			if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {  in __fscache_acquire_cookie()
331 				set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);  in __fscache_acquire_cookie()
334 				fscache_cookie_put(cookie,  in __fscache_acquire_cookie()
341 			set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);  in __fscache_acquire_cookie()
349 	return cookie;  in __fscache_acquire_cookie()
356 void __fscache_enable_cookie(struct fscache_cookie *cookie,  in __fscache_enable_cookie()  argument
362 	_enter("%p", cookie);  in __fscache_enable_cookie()
364 	trace_fscache_enable(cookie);  in __fscache_enable_cookie()
366 	wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,  in __fscache_enable_cookie()
369 	fscache_update_aux(cookie, aux_data);  in __fscache_enable_cookie()
371 	if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))  in __fscache_enable_cookie()
376 	} else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {  in __fscache_enable_cookie()
378 		__fscache_wait_on_invalidate(cookie);  in __fscache_enable_cookie()
380 		if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)  in __fscache_enable_cookie()
381 			set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);  in __fscache_enable_cookie()
383 		set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);  in __fscache_enable_cookie()
387 	clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);  in __fscache_enable_cookie()
388 	wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);  in __fscache_enable_cookie()
397 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,  in fscache_acquire_non_index_cookie()  argument
406 	set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);  in fscache_acquire_non_index_cookie()
419 	cache = fscache_select_cache_for_object(cookie->parent);  in fscache_acquire_non_index_cookie()
429 	set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);  in fscache_acquire_non_index_cookie()
433 	ret = fscache_alloc_object(cache, cookie);  in fscache_acquire_non_index_cookie()
440 	spin_lock(&cookie->lock);  in fscache_acquire_non_index_cookie()
441 	if (hlist_empty(&cookie->backing_objects)) {  in fscache_acquire_non_index_cookie()
442 		spin_unlock(&cookie->lock);  in fscache_acquire_non_index_cookie()
446 	object = hlist_entry(cookie->backing_objects.first,  in fscache_acquire_non_index_cookie()
455 	spin_unlock(&cookie->lock);  in fscache_acquire_non_index_cookie()
459 		_debug("non-deferred lookup %p", &cookie->flags);  in fscache_acquire_non_index_cookie()
460 		wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,  in fscache_acquire_non_index_cookie()
463 		if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))  in fscache_acquire_non_index_cookie()
482 				struct fscache_cookie *cookie)  in fscache_alloc_object()  argument
487 	_enter("%p,%p{%s}", cache, cookie, cookie->def->name);  in fscache_alloc_object()
489 	spin_lock(&cookie->lock);  in fscache_alloc_object()
490 	hlist_for_each_entry(object, &cookie->backing_objects,  in fscache_alloc_object()
495 	spin_unlock(&cookie->lock);  in fscache_alloc_object()
500 	object = cache->ops->alloc_object(cache, cookie);  in fscache_alloc_object()
508 	ASSERTCMP(object->cookie, ==, cookie);  in fscache_alloc_object()
514 	       object->debug_id, cookie->def->name, object->events);  in fscache_alloc_object()
516 	ret = fscache_alloc_object(cache, cookie->parent);  in fscache_alloc_object()
523 	if (fscache_attach_object(cookie, object) < 0) {  in fscache_alloc_object()
536 		spin_unlock(&cookie->lock);  in fscache_alloc_object()
539 	spin_unlock(&cookie->lock);  in fscache_alloc_object()
555 static int fscache_attach_object(struct fscache_cookie *cookie,  in fscache_attach_object()  argument
562 	_enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);  in fscache_attach_object()
564 	ASSERTCMP(object->cookie, ==, cookie);  in fscache_attach_object()
566 	spin_lock(&cookie->lock);  in fscache_attach_object()
571 	hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {  in fscache_attach_object()
580 	spin_lock_nested(&cookie->parent->lock, 1);  in fscache_attach_object()
581 	hlist_for_each_entry(p, &cookie->parent->backing_objects,  in fscache_attach_object()
586 				spin_unlock(&cookie->parent->lock);  in fscache_attach_object()
596 	spin_unlock(&cookie->parent->lock);  in fscache_attach_object()
606 	hlist_add_head(&object->cookie_link, &cookie->backing_objects);  in fscache_attach_object()
612 	spin_unlock(&cookie->lock);  in fscache_attach_object()
620 void __fscache_invalidate(struct fscache_cookie *cookie)  in __fscache_invalidate()  argument
624 	_enter("{%s}", cookie->def->name);  in __fscache_invalidate()
633 	ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);  in __fscache_invalidate()
638 	if (!hlist_empty(&cookie->backing_objects)) {  in __fscache_invalidate()
639 		spin_lock(&cookie->lock);  in __fscache_invalidate()
641 		if (fscache_cookie_enabled(cookie) &&  in __fscache_invalidate()
642 		    !hlist_empty(&cookie->backing_objects) &&  in __fscache_invalidate()
644 				      &cookie->flags)) {  in __fscache_invalidate()
645 			object = hlist_entry(cookie->backing_objects.first,  in __fscache_invalidate()
653 		spin_unlock(&cookie->lock);  in __fscache_invalidate()
663 void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)  in __fscache_wait_on_invalidate()  argument
665 	_enter("%p", cookie);  in __fscache_wait_on_invalidate()
667 	wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,  in __fscache_wait_on_invalidate()
677 void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)  in __fscache_update_cookie()  argument
683 	if (!cookie) {  in __fscache_update_cookie()
689 	_enter("{%s}", cookie->def->name);  in __fscache_update_cookie()
691 	spin_lock(&cookie->lock);  in __fscache_update_cookie()
693 	fscache_update_aux(cookie, aux_data);  in __fscache_update_cookie()
695 	if (fscache_cookie_enabled(cookie)) {  in __fscache_update_cookie()
700 				     &cookie->backing_objects, cookie_link) {  in __fscache_update_cookie()
705 	spin_unlock(&cookie->lock);  in __fscache_update_cookie()
713 void __fscache_disable_cookie(struct fscache_cookie *cookie,  in __fscache_disable_cookie()  argument
720 	_enter("%p,%u", cookie, invalidate);  in __fscache_disable_cookie()
722 	trace_fscache_disable(cookie);  in __fscache_disable_cookie()
724 	ASSERTCMP(atomic_read(&cookie->n_active), >, 0);  in __fscache_disable_cookie()
726 	if (atomic_read(&cookie->n_children) != 0) {  in __fscache_disable_cookie()
728 		       cookie->def->name);  in __fscache_disable_cookie()
732 	wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,  in __fscache_disable_cookie()
735 	fscache_update_aux(cookie, aux_data);  in __fscache_disable_cookie()
737 	if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))  in __fscache_disable_cookie()
743 	__fscache_wait_on_invalidate(cookie);  in __fscache_disable_cookie()
746 	set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);  in __fscache_disable_cookie()
748 	spin_lock(&cookie->lock);  in __fscache_disable_cookie()
749 	if (!hlist_empty(&cookie->backing_objects)) {  in __fscache_disable_cookie()
750 		hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {  in __fscache_disable_cookie()
757 		if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))  in __fscache_disable_cookie()
760 	spin_unlock(&cookie->lock);  in __fscache_disable_cookie()
762 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);  in __fscache_disable_cookie()
768 	if (!atomic_dec_and_test(&cookie->n_active)) {  in __fscache_disable_cookie()
769 		wait_var_event(&cookie->n_active,  in __fscache_disable_cookie()
770 			       !atomic_read(&cookie->n_active));  in __fscache_disable_cookie()
774 	if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)  in __fscache_disable_cookie()
775 		fscache_invalidate_writes(cookie);  in __fscache_disable_cookie()
778 	if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {  in __fscache_disable_cookie()
779 		atomic_inc(&cookie->n_active);  in __fscache_disable_cookie()
780 		set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in __fscache_disable_cookie()
784 	clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);  in __fscache_disable_cookie()
785 	wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);  in __fscache_disable_cookie()
796 void __fscache_relinquish_cookie(struct fscache_cookie *cookie,  in __fscache_relinquish_cookie()  argument
804 	if (!cookie) {  in __fscache_relinquish_cookie()
811 	       cookie, cookie->def->name, cookie->netfs_data,  in __fscache_relinquish_cookie()
812 	       atomic_read(&cookie->n_active), retire);  in __fscache_relinquish_cookie()
814 	trace_fscache_relinquish(cookie, retire);  in __fscache_relinquish_cookie()
817 	if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))  in __fscache_relinquish_cookie()
820 	__fscache_disable_cookie(cookie, aux_data, retire);  in __fscache_relinquish_cookie()
823 	cookie->netfs_data	= NULL;  in __fscache_relinquish_cookie()
824 	cookie->def		= NULL;  in __fscache_relinquish_cookie()
825 	BUG_ON(!radix_tree_empty(&cookie->stores));  in __fscache_relinquish_cookie()
827 	if (cookie->parent) {  in __fscache_relinquish_cookie()
828 		ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);  in __fscache_relinquish_cookie()
829 		ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);  in __fscache_relinquish_cookie()
830 		atomic_dec(&cookie->parent->n_children);  in __fscache_relinquish_cookie()
834 	ASSERTCMP(atomic_read(&cookie->usage), >, 0);  in __fscache_relinquish_cookie()
835 	fscache_cookie_put(cookie, fscache_cookie_put_relinquish);  in __fscache_relinquish_cookie()
844 static void fscache_unhash_cookie(struct fscache_cookie *cookie)  in fscache_unhash_cookie()  argument
849 	bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);  in fscache_unhash_cookie()
853 	hlist_bl_del(&cookie->hash_link);  in fscache_unhash_cookie()
860 void fscache_cookie_put(struct fscache_cookie *cookie,  in fscache_cookie_put()  argument
866 	_enter("%p", cookie);  in fscache_cookie_put()
869 		usage = atomic_dec_return(&cookie->usage);  in fscache_cookie_put()
870 		trace_fscache_cookie(cookie, where, usage);  in fscache_cookie_put()
876 		parent = cookie->parent;  in fscache_cookie_put()
877 		fscache_unhash_cookie(cookie);  in fscache_cookie_put()
878 		fscache_free_cookie(cookie);  in fscache_cookie_put()
880 		cookie = parent;  in fscache_cookie_put()
882 	} while (cookie);  in fscache_cookie_put()
892 int __fscache_check_consistency(struct fscache_cookie *cookie,  in __fscache_check_consistency()  argument
900 	_enter("%p,", cookie);  in __fscache_check_consistency()
902 	ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);  in __fscache_check_consistency()
904 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_check_consistency()
907 	if (hlist_empty(&cookie->backing_objects))  in __fscache_check_consistency()
914 	fscache_operation_init(cookie, op, NULL, NULL, NULL);  in __fscache_check_consistency()
918 	trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);  in __fscache_check_consistency()
920 	spin_lock(&cookie->lock);  in __fscache_check_consistency()
922 	fscache_update_aux(cookie, aux_data);  in __fscache_check_consistency()
924 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_check_consistency()
925 	    hlist_empty(&cookie->backing_objects))  in __fscache_check_consistency()
927 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_check_consistency()
934 	__fscache_use_cookie(cookie);  in __fscache_check_consistency()
939 	spin_unlock(&cookie->lock);  in __fscache_check_consistency()
955 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_check_consistency()
957 	spin_unlock(&cookie->lock);  in __fscache_check_consistency()
959 		__fscache_wake_unused_cookie(cookie);  in __fscache_check_consistency()