19 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)  in __fscache_check_page_write()  argument
24 	val = radix_tree_lookup(&cookie->stores, page->index);  in __fscache_check_page_write()
26 	trace_fscache_check_page(cookie, page, val, 0);  in __fscache_check_page_write()
35 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)  in __fscache_wait_on_page_write()  argument
37 	wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);  in __fscache_wait_on_page_write()
39 	trace_fscache_page(cookie, page, fscache_page_write_wait);  in __fscache_wait_on_page_write()
41 	wait_event(*wq, !__fscache_check_page_write(cookie, page));  in __fscache_wait_on_page_write()
50 bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page)  in release_page_wait_timeout()  argument
52 	wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);  in release_page_wait_timeout()
54 	return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page),  in release_page_wait_timeout()
62 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,  in __fscache_maybe_release_page()  argument
69 	_enter("%p,%p,%x", cookie, page, gfp);  in __fscache_maybe_release_page()
71 	trace_fscache_page(cookie, page, fscache_page_maybe_release);  in __fscache_maybe_release_page()
75 	val = radix_tree_lookup(&cookie->stores, page->index);  in __fscache_maybe_release_page()
79 		__fscache_uncache_page(cookie, page);  in __fscache_maybe_release_page()
85 	if (radix_tree_tag_get(&cookie->stores, page->index,  in __fscache_maybe_release_page()
93 	spin_lock(&cookie->stores_lock);  in __fscache_maybe_release_page()
96 	if (radix_tree_tag_get(&cookie->stores, page->index,  in __fscache_maybe_release_page()
100 		spin_unlock(&cookie->stores_lock);  in __fscache_maybe_release_page()
104 	xpage = radix_tree_delete(&cookie->stores, page->index);  in __fscache_maybe_release_page()
105 	trace_fscache_page(cookie, page, fscache_page_radix_delete);  in __fscache_maybe_release_page()
106 	spin_unlock(&cookie->stores_lock);  in __fscache_maybe_release_page()
116 	wake_up_bit(&cookie->flags, 0);  in __fscache_maybe_release_page()
117 	trace_fscache_wake_cookie(cookie);  in __fscache_maybe_release_page()
120 	__fscache_uncache_page(cookie, page);  in __fscache_maybe_release_page()
134 	if (!release_page_wait_timeout(cookie, page))  in __fscache_maybe_release_page()
149 	struct fscache_cookie *cookie;  in fscache_end_page_write()  local
153 	cookie = object->cookie;  in fscache_end_page_write()
154 	if (cookie) {  in fscache_end_page_write()
157 		spin_lock(&cookie->stores_lock);  in fscache_end_page_write()
158 		radix_tree_tag_clear(&cookie->stores, page->index,  in fscache_end_page_write()
160 		trace_fscache_page(cookie, page, fscache_page_radix_clear_store);  in fscache_end_page_write()
161 		if (!radix_tree_tag_get(&cookie->stores, page->index,  in fscache_end_page_write()
164 			xpage = radix_tree_delete(&cookie->stores, page->index);  in fscache_end_page_write()
165 			trace_fscache_page(cookie, page, fscache_page_radix_delete);  in fscache_end_page_write()
166 			trace_fscache_page(cookie, page, fscache_page_write_end);  in fscache_end_page_write()
168 			val = radix_tree_lookup(&cookie->stores, page->index);  in fscache_end_page_write()
169 			trace_fscache_check_page(cookie, page, val, 1);  in fscache_end_page_write()
171 			trace_fscache_page(cookie, page, fscache_page_write_end_pend);  in fscache_end_page_write()
173 		spin_unlock(&cookie->stores_lock);  in fscache_end_page_write()
174 		wake_up_bit(&cookie->flags, 0);  in fscache_end_page_write()
175 		trace_fscache_wake_cookie(cookie);  in fscache_end_page_write()
177 		trace_fscache_page(cookie, page, fscache_page_write_end_noc);  in fscache_end_page_write()
213 int __fscache_attr_changed(struct fscache_cookie *cookie)  in __fscache_attr_changed()  argument
219 	_enter("%p", cookie);  in __fscache_attr_changed()
221 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_attr_changed()
232 	fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL);  in __fscache_attr_changed()
233 	trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed);  in __fscache_attr_changed()
238 	spin_lock(&cookie->lock);  in __fscache_attr_changed()
240 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_attr_changed()
241 	    hlist_empty(&cookie->backing_objects))  in __fscache_attr_changed()
243 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_attr_changed()
246 	__fscache_use_cookie(cookie);  in __fscache_attr_changed()
249 	spin_unlock(&cookie->lock);  in __fscache_attr_changed()
256 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_attr_changed()
258 	spin_unlock(&cookie->lock);  in __fscache_attr_changed()
261 		__fscache_wake_unused_cookie(cookie);  in __fscache_attr_changed()
293 		fscache_put_context(op->cookie, op->context);  in fscache_release_retrieval_op()
302 	struct fscache_cookie *cookie,  in fscache_alloc_retrieval()  argument
316 	fscache_operation_init(cookie, &op->op, NULL,  in fscache_alloc_retrieval()
322 	op->cookie	= cookie;  in fscache_alloc_retrieval()
332 		fscache_get_context(op->cookie, context);  in fscache_alloc_retrieval()
339 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)  in fscache_wait_for_deferred_lookup()  argument
343 	if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {  in fscache_wait_for_deferred_lookup()
350 	if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,  in fscache_wait_for_deferred_lookup()
357 	ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));  in fscache_wait_for_deferred_lookup()
382 		trace_fscache_op(object->cookie, op, fscache_op_signal);  in fscache_wait_for_operation_activation()
404 		trace_fscache_op(object->cookie, op, fscache_op_signal);  in fscache_wait_for_operation_activation()
423 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,  in __fscache_read_or_alloc_page()  argument
434 	_enter("%p,%p,,,", cookie, page);  in __fscache_read_or_alloc_page()
438 	if (hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_page()
441 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_read_or_alloc_page()
446 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_read_or_alloc_page()
449 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_read_or_alloc_page()
452 	op = fscache_alloc_retrieval(cookie, page->mapping,  in __fscache_read_or_alloc_page()
459 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one);  in __fscache_read_or_alloc_page()
461 	spin_lock(&cookie->lock);  in __fscache_read_or_alloc_page()
463 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_read_or_alloc_page()
464 	    hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_page()
466 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_read_or_alloc_page()
471 	__fscache_use_cookie(cookie);  in __fscache_read_or_alloc_page()
477 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_page()
491 	if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {  in __fscache_read_or_alloc_page()
521 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_read_or_alloc_page()
523 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_page()
525 		__fscache_wake_unused_cookie(cookie);  in __fscache_read_or_alloc_page()
552 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,  in __fscache_read_or_alloc_pages()  argument
565 	_enter("%p,,%d,,,", cookie, *nr_pages);  in __fscache_read_or_alloc_pages()
569 	if (hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_pages()
572 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_read_or_alloc_pages()
577 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_read_or_alloc_pages()
581 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_read_or_alloc_pages()
584 	op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);  in __fscache_read_or_alloc_pages()
588 	trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi);  in __fscache_read_or_alloc_pages()
590 	spin_lock(&cookie->lock);  in __fscache_read_or_alloc_pages()
592 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_read_or_alloc_pages()
593 	    hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_pages()
595 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_read_or_alloc_pages()
598 	__fscache_use_cookie(cookie);  in __fscache_read_or_alloc_pages()
604 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_pages()
618 	if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {  in __fscache_read_or_alloc_pages()
648 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_read_or_alloc_pages()
650 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_pages()
653 		__fscache_wake_unused_cookie(cookie);  in __fscache_read_or_alloc_pages()
669 int __fscache_alloc_page(struct fscache_cookie *cookie,  in __fscache_alloc_page()  argument
678 	_enter("%p,%p,,,", cookie, page);  in __fscache_alloc_page()
682 	if (hlist_empty(&cookie->backing_objects))  in __fscache_alloc_page()
685 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_alloc_page()
688 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_alloc_page()
693 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_alloc_page()
696 	op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);  in __fscache_alloc_page()
700 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one);  in __fscache_alloc_page()
702 	spin_lock(&cookie->lock);  in __fscache_alloc_page()
704 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_alloc_page()
705 	    hlist_empty(&cookie->backing_objects))  in __fscache_alloc_page()
707 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_alloc_page()
710 	__fscache_use_cookie(cookie);  in __fscache_alloc_page()
713 	spin_unlock(&cookie->lock);  in __fscache_alloc_page()
742 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_alloc_page()
744 	spin_unlock(&cookie->lock);  in __fscache_alloc_page()
747 		__fscache_wake_unused_cookie(cookie);  in __fscache_alloc_page()
759 void __fscache_readpages_cancel(struct fscache_cookie *cookie,  in __fscache_readpages_cancel()  argument
766 			__fscache_uncache_page(cookie, page);  in __fscache_readpages_cancel()
787 	struct fscache_cookie *cookie;  in fscache_write_op()  local
797 	cookie = object->cookie;  in fscache_write_op()
810 	if (!cookie) {  in fscache_write_op()
825 	spin_lock(&cookie->stores_lock);  in fscache_write_op()
832 	n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,  in fscache_write_op()
834 	trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit);  in fscache_write_op()
840 	radix_tree_tag_set(&cookie->stores, page->index,  in fscache_write_op()
842 	radix_tree_tag_clear(&cookie->stores, page->index,  in fscache_write_op()
844 	trace_fscache_page(cookie, page, fscache_page_radix_pend2store);  in fscache_write_op()
846 	spin_unlock(&cookie->stores_lock);  in fscache_write_op()
856 	trace_fscache_wrote_page(cookie, page, &op->op, ret);  in fscache_write_op()
870 	trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS);  in fscache_write_op()
878 	spin_unlock(&cookie->stores_lock);  in fscache_write_op()
888 void fscache_invalidate_writes(struct fscache_cookie *cookie)  in fscache_invalidate_writes()  argument
897 		spin_lock(&cookie->stores_lock);  in fscache_invalidate_writes()
898 		n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,  in fscache_invalidate_writes()
902 			spin_unlock(&cookie->stores_lock);  in fscache_invalidate_writes()
908 			radix_tree_delete(&cookie->stores, page->index);  in fscache_invalidate_writes()
909 			trace_fscache_page(cookie, page, fscache_page_radix_delete);  in fscache_invalidate_writes()
910 			trace_fscache_page(cookie, page, fscache_page_inval);  in fscache_invalidate_writes()
913 		spin_unlock(&cookie->stores_lock);  in fscache_invalidate_writes()
919 	wake_up_bit(&cookie->flags, 0);  in fscache_invalidate_writes()
920 	trace_fscache_wake_cookie(cookie);  in fscache_invalidate_writes()
954 int __fscache_write_page(struct fscache_cookie *cookie,  in __fscache_write_page()  argument
964 	_enter("%p,%x,", cookie, (u32) page->flags);  in __fscache_write_page()
966 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_write_page()
971 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_write_page()
980 	fscache_operation_init(cookie, &op->op, fscache_write_op, NULL,  in __fscache_write_page()
990 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one);  in __fscache_write_page()
993 	spin_lock(&cookie->lock);  in __fscache_write_page()
995 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_write_page()
996 	    hlist_empty(&cookie->backing_objects))  in __fscache_write_page()
998 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_write_page()
1003 	trace_fscache_page(cookie, page, fscache_page_write);  in __fscache_write_page()
1012 	spin_lock(&cookie->stores_lock);  in __fscache_write_page()
1016 	ret = radix_tree_insert(&cookie->stores, page->index, page);  in __fscache_write_page()
1024 	trace_fscache_page(cookie, page, fscache_page_radix_insert);  in __fscache_write_page()
1025 	radix_tree_tag_set(&cookie->stores, page->index,  in __fscache_write_page()
1027 	trace_fscache_page(cookie, page, fscache_page_radix_set_pend);  in __fscache_write_page()
1035 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1041 	__fscache_use_cookie(cookie);  in __fscache_write_page()
1045 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1058 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1060 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1068 	spin_lock(&cookie->stores_lock);  in __fscache_write_page()
1069 	radix_tree_delete(&cookie->stores, page->index);  in __fscache_write_page()
1070 	trace_fscache_page(cookie, page, fscache_page_radix_delete);  in __fscache_write_page()
1071 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1072 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_write_page()
1078 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1081 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1085 		__fscache_wake_unused_cookie(cookie);  in __fscache_write_page()
1102 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)  in __fscache_uncache_page()  argument
1108 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_uncache_page()
1117 	trace_fscache_page(cookie, page, fscache_page_uncache);  in __fscache_uncache_page()
1120 	spin_lock(&cookie->lock);  in __fscache_uncache_page()
1122 	if (hlist_empty(&cookie->backing_objects)) {  in __fscache_uncache_page()
1127 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_uncache_page()
1131 	clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in __fscache_uncache_page()
1145 	spin_unlock(&cookie->lock);  in __fscache_uncache_page()
1161 	struct fscache_cookie *cookie = op->op.object->cookie;  in fscache_mark_page_cached()  local
1167 	trace_fscache_page(cookie, page, fscache_page_cached);  in fscache_mark_page_cached()
1175 				cookie->def->name, page->index);  in fscache_mark_page_cached()
1179 	if (cookie->def->mark_page_cached)  in fscache_mark_page_cached()
1180 		cookie->def->mark_page_cached(cookie->netfs_data,  in fscache_mark_page_cached()
1209 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,  in __fscache_uncache_all_inode_pages()  argument
1217 	_enter("%p,%p", cookie, inode);  in __fscache_uncache_all_inode_pages()
1232 				__fscache_wait_on_page_write(cookie, page);  in __fscache_uncache_all_inode_pages()
1233 				__fscache_uncache_page(cookie, page);  in __fscache_uncache_all_inode_pages()