19 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)  in __fscache_check_page_write()  argument
24 	val = radix_tree_lookup(&cookie->stores, page->index);  in __fscache_check_page_write()
26 	trace_fscache_check_page(cookie, page, val, 0);  in __fscache_check_page_write()
35 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)  in __fscache_wait_on_page_write()  argument
37 	wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);  in __fscache_wait_on_page_write()
39 	trace_fscache_page(cookie, page, fscache_page_write_wait);  in __fscache_wait_on_page_write()
41 	wait_event(*wq, !__fscache_check_page_write(cookie, page));  in __fscache_wait_on_page_write()
50 bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page)  in release_page_wait_timeout()  argument
52 	wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);  in release_page_wait_timeout()
54 	return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page),  in release_page_wait_timeout()
62 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,  in __fscache_maybe_release_page()  argument
69 	_enter("%p,%p,%x", cookie, page, gfp);  in __fscache_maybe_release_page()
71 	trace_fscache_page(cookie, page, fscache_page_maybe_release);  in __fscache_maybe_release_page()
75 	val = radix_tree_lookup(&cookie->stores, page->index);  in __fscache_maybe_release_page()
79 		__fscache_uncache_page(cookie, page);  in __fscache_maybe_release_page()
85 	if (radix_tree_tag_get(&cookie->stores, page->index,  in __fscache_maybe_release_page()
93 	spin_lock(&cookie->stores_lock);  in __fscache_maybe_release_page()
96 	if (radix_tree_tag_get(&cookie->stores, page->index,  in __fscache_maybe_release_page()
100 		spin_unlock(&cookie->stores_lock);  in __fscache_maybe_release_page()
104 	xpage = radix_tree_delete(&cookie->stores, page->index);  in __fscache_maybe_release_page()
105 	trace_fscache_page(cookie, page, fscache_page_radix_delete);  in __fscache_maybe_release_page()
106 	spin_unlock(&cookie->stores_lock);  in __fscache_maybe_release_page()
116 	wake_up_bit(&cookie->flags, 0);  in __fscache_maybe_release_page()
117 	trace_fscache_wake_cookie(cookie);  in __fscache_maybe_release_page()
120 	__fscache_uncache_page(cookie, page);  in __fscache_maybe_release_page()
134 	if (!release_page_wait_timeout(cookie, page))  in __fscache_maybe_release_page()
149 	struct fscache_cookie *cookie;  in fscache_end_page_write()  local
153 	cookie = object->cookie;  in fscache_end_page_write()
154 	if (cookie) {  in fscache_end_page_write()
157 		spin_lock(&cookie->stores_lock);  in fscache_end_page_write()
158 		radix_tree_tag_clear(&cookie->stores, page->index,  in fscache_end_page_write()
160 		trace_fscache_page(cookie, page, fscache_page_radix_clear_store);  in fscache_end_page_write()
161 		if (!radix_tree_tag_get(&cookie->stores, page->index,  in fscache_end_page_write()
164 			xpage = radix_tree_delete(&cookie->stores, page->index);  in fscache_end_page_write()
165 			trace_fscache_page(cookie, page, fscache_page_radix_delete);  in fscache_end_page_write()
166 			trace_fscache_page(cookie, page, fscache_page_write_end);  in fscache_end_page_write()
168 			val = radix_tree_lookup(&cookie->stores, page->index);  in fscache_end_page_write()
169 			trace_fscache_check_page(cookie, page, val, 1);  in fscache_end_page_write()
171 			trace_fscache_page(cookie, page, fscache_page_write_end_pend);  in fscache_end_page_write()
173 		spin_unlock(&cookie->stores_lock);  in fscache_end_page_write()
174 		wake_up_bit(&cookie->flags, 0);  in fscache_end_page_write()
175 		trace_fscache_wake_cookie(cookie);  in fscache_end_page_write()
177 		trace_fscache_page(cookie, page, fscache_page_write_end_noc);  in fscache_end_page_write()
213 int __fscache_attr_changed(struct fscache_cookie *cookie)  in __fscache_attr_changed()  argument
219 	_enter("%p", cookie);  in __fscache_attr_changed()
221 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_attr_changed()
232 	fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL);  in __fscache_attr_changed()
233 	trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed);  in __fscache_attr_changed()
238 	spin_lock(&cookie->lock);  in __fscache_attr_changed()
240 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_attr_changed()
241 	    hlist_empty(&cookie->backing_objects))  in __fscache_attr_changed()
243 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_attr_changed()
246 	__fscache_use_cookie(cookie);  in __fscache_attr_changed()
249 	spin_unlock(&cookie->lock);  in __fscache_attr_changed()
256 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_attr_changed()
258 	spin_unlock(&cookie->lock);  in __fscache_attr_changed()
261 		__fscache_wake_unused_cookie(cookie);  in __fscache_attr_changed()
294 		fscache_put_context(op->cookie, op->context);  in fscache_release_retrieval_op()
303 	struct fscache_cookie *cookie,  in fscache_alloc_retrieval()  argument
317 	fscache_operation_init(cookie, &op->op, NULL,  in fscache_alloc_retrieval()
323 	op->cookie	= cookie;  in fscache_alloc_retrieval()
334 		fscache_get_context(op->cookie, context);  in fscache_alloc_retrieval()
341 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)  in fscache_wait_for_deferred_lookup()  argument
347 	if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {  in fscache_wait_for_deferred_lookup()
355 	if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,  in fscache_wait_for_deferred_lookup()
362 	ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));  in fscache_wait_for_deferred_lookup()
388 		trace_fscache_op(object->cookie, op, fscache_op_signal);  in fscache_wait_for_operation_activation()
410 		trace_fscache_op(object->cookie, op, fscache_op_signal);  in fscache_wait_for_operation_activation()
429 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,  in __fscache_read_or_alloc_page()  argument
440 	_enter("%p,%p,,,", cookie, page);  in __fscache_read_or_alloc_page()
444 	if (hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_page()
447 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_read_or_alloc_page()
452 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_read_or_alloc_page()
455 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_read_or_alloc_page()
458 	op = fscache_alloc_retrieval(cookie, page->mapping,  in __fscache_read_or_alloc_page()
465 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one);  in __fscache_read_or_alloc_page()
467 	spin_lock(&cookie->lock);  in __fscache_read_or_alloc_page()
469 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_read_or_alloc_page()
470 	    hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_page()
472 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_read_or_alloc_page()
477 	__fscache_use_cookie(cookie);  in __fscache_read_or_alloc_page()
483 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_page()
497 	if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {  in __fscache_read_or_alloc_page()
527 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_read_or_alloc_page()
529 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_page()
531 		__fscache_wake_unused_cookie(cookie);  in __fscache_read_or_alloc_page()
558 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,  in __fscache_read_or_alloc_pages()  argument
571 	_enter("%p,,%d,,,", cookie, *nr_pages);  in __fscache_read_or_alloc_pages()
575 	if (hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_pages()
578 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_read_or_alloc_pages()
583 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_read_or_alloc_pages()
587 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_read_or_alloc_pages()
590 	op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);  in __fscache_read_or_alloc_pages()
594 	trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi);  in __fscache_read_or_alloc_pages()
596 	spin_lock(&cookie->lock);  in __fscache_read_or_alloc_pages()
598 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_read_or_alloc_pages()
599 	    hlist_empty(&cookie->backing_objects))  in __fscache_read_or_alloc_pages()
601 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_read_or_alloc_pages()
604 	__fscache_use_cookie(cookie);  in __fscache_read_or_alloc_pages()
610 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_pages()
624 	if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {  in __fscache_read_or_alloc_pages()
654 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_read_or_alloc_pages()
656 	spin_unlock(&cookie->lock);  in __fscache_read_or_alloc_pages()
659 		__fscache_wake_unused_cookie(cookie);  in __fscache_read_or_alloc_pages()
675 int __fscache_alloc_page(struct fscache_cookie *cookie,  in __fscache_alloc_page()  argument
684 	_enter("%p,%p,,,", cookie, page);  in __fscache_alloc_page()
688 	if (hlist_empty(&cookie->backing_objects))  in __fscache_alloc_page()
691 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_alloc_page()
694 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_alloc_page()
699 	if (fscache_wait_for_deferred_lookup(cookie) < 0)  in __fscache_alloc_page()
702 	op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);  in __fscache_alloc_page()
706 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one);  in __fscache_alloc_page()
708 	spin_lock(&cookie->lock);  in __fscache_alloc_page()
710 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_alloc_page()
711 	    hlist_empty(&cookie->backing_objects))  in __fscache_alloc_page()
713 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_alloc_page()
716 	__fscache_use_cookie(cookie);  in __fscache_alloc_page()
719 	spin_unlock(&cookie->lock);  in __fscache_alloc_page()
748 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_alloc_page()
750 	spin_unlock(&cookie->lock);  in __fscache_alloc_page()
753 		__fscache_wake_unused_cookie(cookie);  in __fscache_alloc_page()
765 void __fscache_readpages_cancel(struct fscache_cookie *cookie,  in __fscache_readpages_cancel()  argument
772 			__fscache_uncache_page(cookie, page);  in __fscache_readpages_cancel()
793 	struct fscache_cookie *cookie;  in fscache_write_op()  local
803 	cookie = object->cookie;  in fscache_write_op()
816 	if (!cookie) {  in fscache_write_op()
831 	spin_lock(&cookie->stores_lock);  in fscache_write_op()
838 	n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,  in fscache_write_op()
840 	trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit);  in fscache_write_op()
846 	radix_tree_tag_set(&cookie->stores, page->index,  in fscache_write_op()
848 	radix_tree_tag_clear(&cookie->stores, page->index,  in fscache_write_op()
850 	trace_fscache_page(cookie, page, fscache_page_radix_pend2store);  in fscache_write_op()
852 	spin_unlock(&cookie->stores_lock);  in fscache_write_op()
862 	trace_fscache_wrote_page(cookie, page, &op->op, ret);  in fscache_write_op()
876 	trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS);  in fscache_write_op()
884 	spin_unlock(&cookie->stores_lock);  in fscache_write_op()
894 void fscache_invalidate_writes(struct fscache_cookie *cookie)  in fscache_invalidate_writes()  argument
903 		spin_lock(&cookie->stores_lock);  in fscache_invalidate_writes()
904 		n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,  in fscache_invalidate_writes()
908 			spin_unlock(&cookie->stores_lock);  in fscache_invalidate_writes()
914 			radix_tree_delete(&cookie->stores, page->index);  in fscache_invalidate_writes()
915 			trace_fscache_page(cookie, page, fscache_page_radix_delete);  in fscache_invalidate_writes()
916 			trace_fscache_page(cookie, page, fscache_page_inval);  in fscache_invalidate_writes()
919 		spin_unlock(&cookie->stores_lock);  in fscache_invalidate_writes()
925 	wake_up_bit(&cookie->flags, 0);  in fscache_invalidate_writes()
926 	trace_fscache_wake_cookie(cookie);  in fscache_invalidate_writes()
960 int __fscache_write_page(struct fscache_cookie *cookie,  in __fscache_write_page()  argument
970 	_enter("%p,%x,", cookie, (u32) page->flags);  in __fscache_write_page()
972 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_write_page()
977 	if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {  in __fscache_write_page()
986 	fscache_operation_init(cookie, &op->op, fscache_write_op, NULL,  in __fscache_write_page()
996 	trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one);  in __fscache_write_page()
999 	spin_lock(&cookie->lock);  in __fscache_write_page()
1001 	if (!fscache_cookie_enabled(cookie) ||  in __fscache_write_page()
1002 	    hlist_empty(&cookie->backing_objects))  in __fscache_write_page()
1004 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_write_page()
1009 	trace_fscache_page(cookie, page, fscache_page_write);  in __fscache_write_page()
1018 	spin_lock(&cookie->stores_lock);  in __fscache_write_page()
1022 	ret = radix_tree_insert(&cookie->stores, page->index, page);  in __fscache_write_page()
1030 	trace_fscache_page(cookie, page, fscache_page_radix_insert);  in __fscache_write_page()
1031 	radix_tree_tag_set(&cookie->stores, page->index,  in __fscache_write_page()
1033 	trace_fscache_page(cookie, page, fscache_page_radix_set_pend);  in __fscache_write_page()
1041 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1047 	__fscache_use_cookie(cookie);  in __fscache_write_page()
1051 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1064 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1066 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1074 	spin_lock(&cookie->stores_lock);  in __fscache_write_page()
1075 	radix_tree_delete(&cookie->stores, page->index);  in __fscache_write_page()
1076 	trace_fscache_page(cookie, page, fscache_page_radix_delete);  in __fscache_write_page()
1077 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1078 	wake_cookie = __fscache_unuse_cookie(cookie);  in __fscache_write_page()
1084 	spin_unlock(&cookie->stores_lock);  in __fscache_write_page()
1087 	spin_unlock(&cookie->lock);  in __fscache_write_page()
1091 		__fscache_wake_unused_cookie(cookie);  in __fscache_write_page()
1108 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)  in __fscache_uncache_page()  argument
1114 	ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);  in __fscache_uncache_page()
1123 	trace_fscache_page(cookie, page, fscache_page_uncache);  in __fscache_uncache_page()
1126 	spin_lock(&cookie->lock);  in __fscache_uncache_page()
1128 	if (hlist_empty(&cookie->backing_objects)) {  in __fscache_uncache_page()
1133 	object = hlist_entry(cookie->backing_objects.first,  in __fscache_uncache_page()
1137 	clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in __fscache_uncache_page()
1151 	spin_unlock(&cookie->lock);  in __fscache_uncache_page()
1167 	struct fscache_cookie *cookie = op->op.object->cookie;  in fscache_mark_page_cached()  local
1173 	trace_fscache_page(cookie, page, fscache_page_cached);  in fscache_mark_page_cached()
1181 				cookie->def->name, page->index);  in fscache_mark_page_cached()
1185 	if (cookie->def->mark_page_cached)  in fscache_mark_page_cached()
1186 		cookie->def->mark_page_cached(cookie->netfs_data,  in fscache_mark_page_cached()
1215 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,  in __fscache_uncache_all_inode_pages()  argument
1223 	_enter("%p,%p", cookie, inode);  in __fscache_uncache_all_inode_pages()
1238 				__fscache_wait_on_page_write(cookie, page);  in __fscache_uncache_all_inode_pages()
1239 				__fscache_uncache_page(cookie, page);  in __fscache_uncache_all_inode_pages()