23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)  in __fscache_check_page_write()  argument
28 	val = radix_tree_lookup(&cookie->stores, page->index);  in __fscache_check_page_write()
30 	trace_fscache_check_page(cookie, page, val, 0);  in __fscache_check_page_write()
39 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)  in __fscache_wait_on_page_write()  argument
41 	wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);  in __fscache_wait_on_page_write()
43 	trace_fscache_page(cookie, page, fscache_page_write_wait);  in __fscache_wait_on_page_write()
45 	wait_event(*wq, !__fscache_check_page_write(cookie, page));  in __fscache_wait_on_page_write()
54 bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page)  in release_page_wait_timeout()  argument
56 	wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);  in release_page_wait_timeout()
58 	return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page),  in release_page_wait_timeout()
66 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,  in __fscache_maybe_release_page()  argument
73 	_enter("%p,%p,%x", cookie, page, gfp);  in __fscache_maybe_release_page()
75 	trace_fscache_page(cookie, page, fscache_page_maybe_release);  in __fscache_maybe_release_page()
79 	val = radix_tree_lookup(&cookie->stores, page->index);  in __fscache_maybe_release_page()
83 		__fscache_uncache_page(cookie, page);  in __fscache_maybe_release_page()
89 	if (radix_tree_tag_get(&cookie->stores, page->index,  in __fscache_maybe_release_page()
97 	spin_lock(&cookie->stores_lock);  in __fscache_maybe_release_page()
100 	if (radix_tree_tag_get(&cookie->stores, page->index,  in __fscache_maybe_release_page()
104 		spin_unlock(&cookie->stores_lock);  in __fscache_maybe_release_page()
108 	xpage = radix_tree_delete(&cookie->stores, page->index);  in __fscache_maybe_release_page()
109 	trace_fscache_page(cookie, page, fscache_page_radix_delete);  in __fscache_maybe_release_page()
110 	spin_unlock(&cookie->stores_lock);  in __fscache_maybe_release_page()
120 	wake_up_bit(&cookie->flags, 0);  in __fscache_maybe_release_page()
121 	trace_fscache_wake_cookie(cookie);  in __fscache_maybe_release_page()
124 	__fscache_uncache_page(cookie, page);  in __fscache_maybe_release_page()
138 	if (!release_page_wait_timeout(cookie, page))  in __fscache_maybe_release_page()
153 	struct fscache_cookie *cookie;  in fscache_end_page_write()  local
157 	cookie = object->cookie;  in fscache_end_page_write()
158 	if (cookie) {  in fscache_end_page_write()
161 		spin_lock(&cookie->stores_lock);  in fscache_end_page_write()
162 		radix_tree_tag_clear(&cookie->stores, page->index,  in fscache_end_page_write()
164 		trace_fscache_page(cookie, page, fscache_page_radix_clear_store);  in fscache_end_page_write()
165 		if (!radix_tree_tag_get(&cookie->stores, page->index,  in fscache_end_page_write()
168 			xpage = radix_tree_delete(&cookie->stores, page->index);  in fscache_end_page_write()
169 			trace_fscache_page(cookie, page, fscache_page_radix_delete);  in fscache_end_page_write()
170 			trace_fscache_page(cookie, page, fscache_page_write_end);  in fscache_end_page_write()
172 			val = radix_tree_lookup(&cookie->stores, page->index);  in fscache_end_page_write()
173 			trace_fscache_check_page(cookie, page, val, 1);  in fscache_end_page_write()
175 			trace_fscache_page(cookie, page, fscache_page_write_end_pend);  in fscache_end_page_write()
177 		spin_unlock(&cookie->stores_lock);  in fscache_end_page_write()
178 		wake_up_bit(&cookie->flags, 0);  in fscache_end_page_write()
179 		trace_fscache_wake_cookie(cookie);  in fscache_end_page_write()
181 		trace_fscache_page(cookie, page, fscache_page_write_end_noc);  in fscache_end_page_write()
217 int __fscache_attr_changed(struct fscache_cookie *cookie)  in __fscache_attr_changed()  argument
223 	_enter("%p", cookie);  in __fscache_attr_changed()
225 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_attr_changed()
236 	fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL);  in __fscache_attr_changed()
237 	trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed);  in __fscache_attr_changed()
242 	spin_lock(&cookie->lock);  in __fscache_attr_changed()
244 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_attr_changed()
245 	    hlist_empty(&cookie->backing_objects))  in __fscache_attr_changed()
247 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_attr_changed()
250 	__fscache_use_cookie(cookie);  in __fscache_attr_changed()
253 	spin_unlock(&cookie->lock);  in __fscache_attr_changed()
260 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_attr_changed()
262 	spin_unlock(&cookie->lock);  in __fscache_attr_changed()
265 		__fscache_wake_unused_cookie(cookie);  in __fscache_attr_changed()
298 		fscache_put_context(op->cookie, op->context);  in fscache_release_retrieval_op()
307 	struct fscache_cookie *cookie,  in fscache_alloc_retrieval()  argument
321 	fscache_operation_init(cookie, &op->op, NULL,  in fscache_alloc_retrieval()
327 	op->cookie	= cookie;  in fscache_alloc_retrieval()
338 		fscache_get_context(op->cookie, context);  in fscache_alloc_retrieval()
345 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)  in fscache_wait_for_deferred_lookup()  argument
351 	if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {  in fscache_wait_for_deferred_lookup()
359 	if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,  in fscache_wait_for_deferred_lookup()
366 	ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));  in fscache_wait_for_deferred_lookup()
392 		trace_fscache_op(object->cookie, op, fscache_op_signal);  in fscache_wait_for_operation_activation()
414 		trace_fscache_op(object->cookie, op, fscache_op_signal);  in fscache_wait_for_operation_activation()
433 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,  in __fscache_read_or_alloc_page()  argument
444 	_enter("%p,%p,,,", cookie, page);  in __fscache_read_or_alloc_page()
448 	if (hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_page()
451 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_read_or_alloc_page()
456 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_read_or_alloc_page()
459 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_read_or_alloc_page()
462 	op = fscache_alloc_retrieval(cookie, page->mapping,  in __fscache_read_or_alloc_page()
469 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one);  in __fscache_read_or_alloc_page()
471 	spin_lock(&cookie->lock);  in __fscache_read_or_alloc_page()
473 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_read_or_alloc_page()
474 	    hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_page()
476 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_read_or_alloc_page()
481 	__fscache_use_cookie(cookie);  in __fscache_read_or_alloc_page()
487 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_page()
501 	if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {  in __fscache_read_or_alloc_page()
531 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_read_or_alloc_page()
533 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_page()
535 		__fscache_wake_unused_cookie(cookie);  in __fscache_read_or_alloc_page()
562 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,  in __fscache_read_or_alloc_pages()  argument
575 	_enter("%p,,%d,,,", cookie, *nr_pages);  in __fscache_read_or_alloc_pages()
579 	if (hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_pages()
582 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_read_or_alloc_pages()
587 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_read_or_alloc_pages()
591 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_read_or_alloc_pages()
594 	op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);  in __fscache_read_or_alloc_pages()
598 	trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi);  in __fscache_read_or_alloc_pages()
600 	spin_lock(&cookie->lock);  in __fscache_read_or_alloc_pages()
602 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_read_or_alloc_pages()
603 	    hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_pages()
605 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_read_or_alloc_pages()
608 	__fscache_use_cookie(cookie);  in __fscache_read_or_alloc_pages()
614 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_pages()
628 	if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {  in __fscache_read_or_alloc_pages()
658 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_read_or_alloc_pages()
660 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_pages()
663 		__fscache_wake_unused_cookie(cookie);  in __fscache_read_or_alloc_pages()
679 int __fscache_alloc_page(struct fscache_cookie *cookie,  in __fscache_alloc_page()  argument
688 	_enter("%p,%p,,,", cookie, page);  in __fscache_alloc_page()
692 	if (hlist_empty(&cookie->backing_objects))  in __fscache_alloc_page()
695 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_alloc_page()
698 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_alloc_page()
703 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_alloc_page()
706 	op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);  in __fscache_alloc_page()
710 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one);  in __fscache_alloc_page()
712 	spin_lock(&cookie->lock);  in __fscache_alloc_page()
714 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_alloc_page()
715 	    hlist_empty(&cookie->backing_objects))  in __fscache_alloc_page()
717 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_alloc_page()
720 	__fscache_use_cookie(cookie);  in __fscache_alloc_page()
723 	spin_unlock(&cookie->lock);  in __fscache_alloc_page()
752 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_alloc_page()
754 	spin_unlock(&cookie->lock);  in __fscache_alloc_page()
757 		__fscache_wake_unused_cookie(cookie);  in __fscache_alloc_page()
769 void __fscache_readpages_cancel(struct fscache_cookie *cookie,  in __fscache_readpages_cancel()  argument
776 			__fscache_uncache_page(cookie, page);  in __fscache_readpages_cancel()
797 	struct fscache_cookie *cookie;  in fscache_write_op()  local
807 	cookie = object->cookie;  in fscache_write_op()
820 	if (!cookie) {  in fscache_write_op()
835 	spin_lock(&cookie->stores_lock);  in fscache_write_op()
842 	n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,  in fscache_write_op()
844 	trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit);  in fscache_write_op()
850 	radix_tree_tag_set(&cookie->stores, page->index,  in fscache_write_op()
852 	radix_tree_tag_clear(&cookie->stores, page->index,  in fscache_write_op()
854 	trace_fscache_page(cookie, page, fscache_page_radix_pend2store);  in fscache_write_op()
856 	spin_unlock(&cookie->stores_lock);  in fscache_write_op()
866 	trace_fscache_wrote_page(cookie, page, &op->op, ret);  in fscache_write_op()
880 	trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS);  in fscache_write_op()
888 	spin_unlock(&cookie->stores_lock);  in fscache_write_op()
898 void fscache_invalidate_writes(struct fscache_cookie *cookie)  in fscache_invalidate_writes()  argument
907 		spin_lock(&cookie->stores_lock);  in fscache_invalidate_writes()
908 		n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,  in fscache_invalidate_writes()
912 			spin_unlock(&cookie->stores_lock);  in fscache_invalidate_writes()
918 			radix_tree_delete(&cookie->stores, page->index);  in fscache_invalidate_writes()
919 			trace_fscache_page(cookie, page, fscache_page_radix_delete);  in fscache_invalidate_writes()
920 			trace_fscache_page(cookie, page, fscache_page_inval);  in fscache_invalidate_writes()
923 		spin_unlock(&cookie->stores_lock);  in fscache_invalidate_writes()
929 	wake_up_bit(&cookie->flags, 0);  in fscache_invalidate_writes()
930 	trace_fscache_wake_cookie(cookie);  in fscache_invalidate_writes()
964 int __fscache_write_page(struct fscache_cookie *cookie,  in __fscache_write_page()  argument
974 	_enter("%p,%x,", cookie, (u32) page->flags);  in __fscache_write_page()
976 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_write_page()
981 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_write_page()
990 	fscache_operation_init(cookie, &op->op, fscache_write_op, NULL,  in __fscache_write_page()
1000 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one);  in __fscache_write_page()
1003 	spin_lock(&cookie->lock);  in __fscache_write_page()
1005 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_write_page()
1006 	    hlist_empty(&cookie->backing_objects))  in __fscache_write_page()
1008 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_write_page()
1013 	trace_fscache_page(cookie, page, fscache_page_write);  in __fscache_write_page()
1022 	spin_lock(&cookie->stores_lock);  in __fscache_write_page()
1026 	ret = radix_tree_insert(&cookie->stores, page->index, page);  in __fscache_write_page()
1034 	trace_fscache_page(cookie, page, fscache_page_radix_insert);  in __fscache_write_page()
1035 	radix_tree_tag_set(&cookie->stores, page->index,  in __fscache_write_page()
1037 	trace_fscache_page(cookie, page, fscache_page_radix_set_pend);  in __fscache_write_page()
1045 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1051 	__fscache_use_cookie(cookie);  in __fscache_write_page()
1055 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1068 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1070 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1078 	spin_lock(&cookie->stores_lock);  in __fscache_write_page()
1079 	radix_tree_delete(&cookie->stores, page->index);  in __fscache_write_page()
1080 	trace_fscache_page(cookie, page, fscache_page_radix_delete);  in __fscache_write_page()
1081 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1082 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_write_page()
1088 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1091 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1095 		__fscache_wake_unused_cookie(cookie);  in __fscache_write_page()
1112 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)  in __fscache_uncache_page()  argument
1118 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_uncache_page()
1127 	trace_fscache_page(cookie, page, fscache_page_uncache);  in __fscache_uncache_page()
1130 	spin_lock(&cookie->lock);  in __fscache_uncache_page()
1132 	if (hlist_empty(&cookie->backing_objects)) {  in __fscache_uncache_page()
1137 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_uncache_page()
1141 	clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in __fscache_uncache_page()
1155 	spin_unlock(&cookie->lock);  in __fscache_uncache_page()
1171 	struct fscache_cookie *cookie = op->op.object->cookie;  in fscache_mark_page_cached()  local
1177 	trace_fscache_page(cookie, page, fscache_page_cached);  in fscache_mark_page_cached()
1185 				cookie->def->name, page->index);  in fscache_mark_page_cached()
1189 	if (cookie->def->mark_page_cached)  in fscache_mark_page_cached()
1190 		cookie->def->mark_page_cached(cookie->netfs_data,  in fscache_mark_page_cached()
1219 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,  in __fscache_uncache_all_inode_pages()  argument
1227 	_enter("%p,%p", cookie, inode);  in __fscache_uncache_all_inode_pages()
1242 				__fscache_wait_on_page_write(cookie, page);  in __fscache_uncache_all_inode_pages()
1243 				__fscache_uncache_page(cookie, page);  in __fscache_uncache_all_inode_pages()