1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* FS-Cache object state machine handler
8  * object state machine and the in-kernel representations.
32 #define STATE(n) (&__STATE_NAME(n))  macro
35  * Define a work state.  Work states are execution states.  No event processing
36  * is performed by them.  The function attached to a work state returns a
37  * pointer indicating the next state to which the state machine should
38  * transition.  Returning NO_TRANSIT repeats the current state, but goes back
51 #define transit_to(state) ({ prefetch(&STATE(state)->work); STATE(state); })  argument
56  * Define a wait state.  Wait states are event processing states.  No execution
57  * is performed by them.  Wait states are just tables of "if event X occurs,
58  * clear it and transition to state Y".  The dispatcher returns to the
59  * scheduler if none of the events in which the wait state has an interest are
70 #define TRANSIT_TO(state, emask) \  argument
71 	{ .events = (emask), .transit_to = STATE(state) }
74  * The object state machine.
107  * Out-of-band event transition tables.  These are for handling unexpected
108  * events, such as an I/O error.  If an OOB event occurs, the state machine
110  * state (acurrently executing work states will complete first).
112  * In such a situation, object->state remembers the state the machine should
150 	struct fscache_object *parent = object->parent;  in fscache_done_parent_op()
153 	       object->debug_id, parent->debug_id, parent->n_ops);  in fscache_done_parent_op()
155 	spin_lock_nested(&parent->lock, 1);  in fscache_done_parent_op()
156 	parent->n_obj_ops--;  in fscache_done_parent_op()
157 	parent->n_ops--;  in fscache_done_parent_op()
158 	if (parent->n_ops == 0)  in fscache_done_parent_op()
160 	spin_unlock(&parent->lock);  in fscache_done_parent_op()
164  * Object state machine dispatcher.
169 	const struct fscache_state *state, *new_state;  in fscache_object_sm_dispatcher()  local
172 	int event = -1;  in fscache_object_sm_dispatcher()
177 	       object->debug_id, object->state->name, object->events);  in fscache_object_sm_dispatcher()
179 	event_mask = object->event_mask;  in fscache_object_sm_dispatcher()
181 	object->event_mask = 0; /* Mask normal event handling */  in fscache_object_sm_dispatcher()
182 	state = object->state;  in fscache_object_sm_dispatcher()
184 	events = object->events;  in fscache_object_sm_dispatcher()
186 	/* Handle any out-of-band events (typically an error) */  in fscache_object_sm_dispatcher()
187 	if (events & object->oob_event_mask) {  in fscache_object_sm_dispatcher()
189 		       object->debug_id, events & object->oob_event_mask);  in fscache_object_sm_dispatcher()
191 		for (t = object->oob_table; t->events; t++) {  in fscache_object_sm_dispatcher()
192 			if (events & t->events) {  in fscache_object_sm_dispatcher()
193 				state = t->transit_to;  in fscache_object_sm_dispatcher()
194 				ASSERT(state->work != NULL);  in fscache_object_sm_dispatcher()
195 				event = fls(events & t->events) - 1;  in fscache_object_sm_dispatcher()
196 				__clear_bit(event, &object->oob_event_mask);  in fscache_object_sm_dispatcher()
197 				clear_bit(event, &object->events);  in fscache_object_sm_dispatcher()
204 	/* Wait states are just transition tables */  in fscache_object_sm_dispatcher()
205 	if (!state->work) {  in fscache_object_sm_dispatcher()
207 			for (t = state->transitions; t->events; t++) {  in fscache_object_sm_dispatcher()
208 				if (events & t->events) {  in fscache_object_sm_dispatcher()
209 					new_state = t->transit_to;  in fscache_object_sm_dispatcher()
210 					event = fls(events & t->events) - 1;  in fscache_object_sm_dispatcher()
211 					trace_fscache_osm(object, state,  in fscache_object_sm_dispatcher()
213 					clear_bit(event, &object->events);  in fscache_object_sm_dispatcher()
214 					_debug("{OBJ%x} ev %d: %s -> %s",  in fscache_object_sm_dispatcher()
215 					       object->debug_id, event,  in fscache_object_sm_dispatcher()
216 					       state->name, new_state->name);  in fscache_object_sm_dispatcher()
217 					object->state = state = new_state;  in fscache_object_sm_dispatcher()
230 	_debug("{OBJ%x} exec %s", object->debug_id, state->name);  in fscache_object_sm_dispatcher()
232 	trace_fscache_osm(object, state, false, oob, event);  in fscache_object_sm_dispatcher()
233 	new_state = state->work(object, event);  in fscache_object_sm_dispatcher()
234 	event = -1;  in fscache_object_sm_dispatcher()
236 		_debug("{OBJ%x} %s notrans", object->debug_id, state->name);  in fscache_object_sm_dispatcher()
237 		if (unlikely(state == STATE(OBJECT_DEAD))) {  in fscache_object_sm_dispatcher()
242 		event_mask = object->oob_event_mask;  in fscache_object_sm_dispatcher()
246 	_debug("{OBJ%x} %s -> %s",  in fscache_object_sm_dispatcher()
247 	       object->debug_id, state->name, new_state->name);  in fscache_object_sm_dispatcher()
248 	object->state = state = new_state;  in fscache_object_sm_dispatcher()
250 	if (state->work) {  in fscache_object_sm_dispatcher()
251 		if (unlikely(state == STATE(OBJECT_DEAD))) {  in fscache_object_sm_dispatcher()
258 	/* Transited to wait state */  in fscache_object_sm_dispatcher()
259 	event_mask = object->oob_event_mask;  in fscache_object_sm_dispatcher()
260 	for (t = state->transitions; t->events; t++)  in fscache_object_sm_dispatcher()
261 		event_mask |= t->events;  in fscache_object_sm_dispatcher()
264 	object->event_mask = event_mask;  in fscache_object_sm_dispatcher()
266 	events = object->events;  in fscache_object_sm_dispatcher()
280 	_enter("{OBJ%x}", object->debug_id);  in fscache_object_work_func()
287  * fscache_object_init - Initialise a cache object description
294  * See Documentation/filesystems/caching/backend-api.rst for a complete
303 	atomic_inc(&cache->object_count);  in fscache_object_init()
305 	object->state = STATE(WAIT_FOR_INIT);  in fscache_object_init()
306 	object->oob_table = fscache_osm_init_oob;  in fscache_object_init()
307 	object->flags = 1 << FSCACHE_OBJECT_IS_LIVE;  in fscache_object_init()
308 	spin_lock_init(&object->lock);  in fscache_object_init()
309 	INIT_LIST_HEAD(&object->cache_link);  in fscache_object_init()
310 	INIT_HLIST_NODE(&object->cookie_link);  in fscache_object_init()
311 	INIT_WORK(&object->work, fscache_object_work_func);  in fscache_object_init()
312 	INIT_LIST_HEAD(&object->dependents);  in fscache_object_init()
313 	INIT_LIST_HEAD(&object->dep_link);  in fscache_object_init()
314 	INIT_LIST_HEAD(&object->pending_ops);  in fscache_object_init()
315 	object->n_children = 0;  in fscache_object_init()
316 	object->n_ops = object->n_in_progress = object->n_exclusive = 0;  in fscache_object_init()
317 	object->events = 0;  in fscache_object_init()
318 	object->store_limit = 0;  in fscache_object_init()
319 	object->store_limit_l = 0;  in fscache_object_init()
320 	object->cache = cache;  in fscache_object_init()
321 	object->cookie = cookie;  in fscache_object_init()
323 	object->parent = NULL;  in fscache_object_init()
325 	RB_CLEAR_NODE(&object->objlist_link);  in fscache_object_init()
328 	object->oob_event_mask = 0;  in fscache_object_init()
329 	for (t = object->oob_table; t->events; t++)  in fscache_object_init()
330 		object->oob_event_mask |= t->events;  in fscache_object_init()
331 	object->event_mask = object->oob_event_mask;  in fscache_object_init()
332 	for (t = object->state->transitions; t->events; t++)  in fscache_object_init()
333 		object->event_mask |= t->events;  in fscache_object_init()
343 	spin_lock(&object->lock);  in fscache_mark_object_dead()
344 	clear_bit(FSCACHE_OBJECT_IS_LIVE, &object->flags);  in fscache_mark_object_dead()
345 	spin_unlock(&object->lock);  in fscache_mark_object_dead()
354 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_abort_initialisation()
356 	object->oob_event_mask = 0;  in fscache_abort_initialisation()
363  * - check the specified object's parent to see if we can make use of it
365  * - we may need to start the process of creating a parent and we need to wait
374 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_initialise_object()
376 	ASSERT(list_empty(&object->dep_link));  in fscache_initialise_object()
378 	parent = object->parent;  in fscache_initialise_object()
384 	_debug("parent: %s of:%lx", parent->state->name, parent->flags);  in fscache_initialise_object()
396 	_debug("wait");  in fscache_initialise_object()
398 	spin_lock(&parent->lock);  in fscache_initialise_object()
402 	    object->cache->ops->grab_object(object, fscache_obj_get_add_to_deps)) {  in fscache_initialise_object()
403 		list_add(&object->dep_link, &parent->dependents);  in fscache_initialise_object()
407 	spin_unlock(&parent->lock);  in fscache_initialise_object()
416 	_leave(" [wait]");  in fscache_initialise_object()
426 	struct fscache_object *parent = object->parent;  in fscache_parent_ready()
428 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_parent_ready()
432 	spin_lock(&parent->lock);  in fscache_parent_ready()
433 	parent->n_ops++;  in fscache_parent_ready()
434 	parent->n_obj_ops++;  in fscache_parent_ready()
435 	spin_unlock(&parent->lock);  in fscache_parent_ready()
443  * - we hold an "access lock" on the parent object, so the parent object cannot
449 	struct fscache_cookie *cookie = object->cookie;  in fscache_look_up_object()
450 	struct fscache_object *parent = object->parent;  in fscache_look_up_object()
453 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_look_up_object()
455 	object->oob_table = fscache_osm_lookup_oob;  in fscache_look_up_object()
458 	ASSERTCMP(parent->n_ops, >, 0);  in fscache_look_up_object()
459 	ASSERTCMP(parent->n_obj_ops, >, 0);  in fscache_look_up_object()
465 	    test_bit(FSCACHE_IOERROR, &object->cache->flags) ||  in fscache_look_up_object()
472 	       cookie->def->name, object->cache->tag->name);  in fscache_look_up_object()
476 	ret = object->cache->ops->lookup_object(object);  in fscache_look_up_object()
481 	if (ret == -ETIMEDOUT) {  in fscache_look_up_object()
499  * fscache_object_lookup_negative - Note negative cookie lookup
507 	struct fscache_cookie *cookie = object->cookie;  in fscache_object_lookup_negative()
509 	_enter("{OBJ%x,%s}", object->debug_id, object->state->name);  in fscache_object_lookup_negative()
511 	if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {  in fscache_object_lookup_negative()
517 		set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in fscache_object_lookup_negative()
518 		clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);  in fscache_object_lookup_negative()
520 		clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);  in fscache_object_lookup_negative()
521 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_object_lookup_negative()
528  * fscache_obtained_object - Note successful object lookup or creation
539 	struct fscache_cookie *cookie = object->cookie;  in fscache_obtained_object()
541 	_enter("{OBJ%x,%s}", object->debug_id, object->state->name);  in fscache_obtained_object()
545 	if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {  in fscache_obtained_object()
549 		clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in fscache_obtained_object()
550 		clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);  in fscache_obtained_object()
555 		clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);  in fscache_obtained_object()
556 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_obtained_object()
561 	set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags);  in fscache_obtained_object()
572 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_object_available()
574 	object->oob_table = fscache_osm_run_oob;  in fscache_object_available()
576 	spin_lock(&object->lock);  in fscache_object_available()
579 	if (object->n_in_progress == 0) {  in fscache_object_available()
580 		if (object->n_ops > 0) {  in fscache_object_available()
581 			ASSERTCMP(object->n_ops, >=, object->n_obj_ops);  in fscache_object_available()
584 			ASSERT(list_empty(&object->pending_ops));  in fscache_object_available()
587 	spin_unlock(&object->lock);  in fscache_object_available()
590 	object->cache->ops->lookup_complete(object);  in fscache_object_available()
605 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_jumpstart_dependents()
620 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_lookup_failure()
622 	object->oob_event_mask = 0;  in fscache_lookup_failure()
625 	object->cache->ops->lookup_complete(object);  in fscache_lookup_failure()
628 	set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags);  in fscache_lookup_failure()
630 	cookie = object->cookie;  in fscache_lookup_failure()
631 	set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);  in fscache_lookup_failure()
632 	if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags))  in fscache_lookup_failure()
633 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_lookup_failure()
640  * Wait for completion of all active operations on this object and the death of
647 	       object->debug_id, object->n_ops, object->n_children, event);  in fscache_kill_object()
650 	object->oob_event_mask = 0;  in fscache_kill_object()
652 	if (test_bit(FSCACHE_OBJECT_RETIRED, &object->flags)) {  in fscache_kill_object()
654 		clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);  in fscache_kill_object()
658 	if (list_empty(&object->dependents) &&  in fscache_kill_object()
659 	    object->n_ops == 0 &&  in fscache_kill_object()
660 	    object->n_children == 0)  in fscache_kill_object()
663 	if (object->n_in_progress == 0) {  in fscache_kill_object()
664 		spin_lock(&object->lock);  in fscache_kill_object()
665 		if (object->n_ops > 0 && object->n_in_progress == 0)  in fscache_kill_object()
667 		spin_unlock(&object->lock);  in fscache_kill_object()
670 	if (!list_empty(&object->dependents))  in fscache_kill_object()
682 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_kill_dependents()
695 	struct fscache_object *parent = object->parent;  in fscache_drop_object()
696 	struct fscache_cookie *cookie = object->cookie;  in fscache_drop_object()
697 	struct fscache_cache *cache = object->cache;  in fscache_drop_object()
700 	_enter("{OBJ%x,%d},%d", object->debug_id, object->n_children, event);  in fscache_drop_object()
703 	ASSERT(!hlist_unhashed(&object->cookie_link));  in fscache_drop_object()
705 	if (test_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags)) {  in fscache_drop_object()
713 	spin_lock(&cookie->lock);  in fscache_drop_object()
714 	hlist_del_init(&object->cookie_link);  in fscache_drop_object()
715 	if (hlist_empty(&cookie->backing_objects) &&  in fscache_drop_object()
716 	    test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))  in fscache_drop_object()
718 	spin_unlock(&cookie->lock);  in fscache_drop_object()
721 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);  in fscache_drop_object()
722 	if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags))  in fscache_drop_object()
723 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_drop_object()
729 	spin_lock(&object->lock);  in fscache_drop_object()
730 	spin_unlock(&object->lock);  in fscache_drop_object()
733 	spin_lock(&cache->object_list_lock);  in fscache_drop_object()
734 	list_del_init(&object->cache_link);  in fscache_drop_object()
735 	spin_unlock(&cache->object_list_lock);  in fscache_drop_object()
738 	cache->ops->drop_object(object);  in fscache_drop_object()
744 		       parent->debug_id, parent->n_children);  in fscache_drop_object()
746 		spin_lock(&parent->lock);  in fscache_drop_object()
747 		parent->n_children--;  in fscache_drop_object()
748 		if (parent->n_children == 0)  in fscache_drop_object()
750 		spin_unlock(&parent->lock);  in fscache_drop_object()
751 		object->parent = NULL;  in fscache_drop_object()
771 	ret = object->cache->ops->grab_object(object, why) ? 0 : -EAGAIN;  in fscache_get_object()
783 	object->cache->ops->put_object(object, why);  in fscache_put_object()
788  * fscache_object_destroy - Note that a cache object is about to be destroyed
796 	fscache_cookie_put(object->cookie, fscache_cookie_put_object);  in fscache_object_destroy()
797 	object->cookie = NULL;  in fscache_object_destroy()
802  * enqueue an object for metadata-type processing
806 	_enter("{OBJ%x}", object->debug_id);  in fscache_enqueue_object()
812 		if (queue_work(fscache_object_wq, &object->work)) {  in fscache_enqueue_object()
823  * fscache_object_sleep_till_congested - Sleep until object wq is congested
837 	DEFINE_WAIT(wait);  in fscache_object_sleep_till_congested()
842 	add_wait_queue_exclusive(cong_wq, &wait);  in fscache_object_sleep_till_congested()
845 	finish_wait(cong_wq, &wait);  in fscache_object_sleep_till_congested()
852  * Enqueue the dependents of an object for metadata-type processing.
863 	_enter("{OBJ%x}", object->debug_id);  in fscache_enqueue_dependents()
865 	if (list_empty(&object->dependents))  in fscache_enqueue_dependents()
868 	spin_lock(&object->lock);  in fscache_enqueue_dependents()
870 	while (!list_empty(&object->dependents)) {  in fscache_enqueue_dependents()
871 		dep = list_entry(object->dependents.next,  in fscache_enqueue_dependents()
873 		list_del_init(&dep->dep_link);  in fscache_enqueue_dependents()
878 		if (!list_empty(&object->dependents) && need_resched()) {  in fscache_enqueue_dependents()
884 	spin_unlock(&object->lock);  in fscache_enqueue_dependents()
893 	_enter("{OBJ%x}", object->debug_id);  in fscache_dequeue_object()
895 	if (!list_empty(&object->dep_link)) {  in fscache_dequeue_object()
896 		spin_lock(&object->parent->lock);  in fscache_dequeue_object()
897 		list_del_init(&object->dep_link);  in fscache_dequeue_object()
898 		spin_unlock(&object->parent->lock);  in fscache_dequeue_object()
905  * fscache_check_aux - Ask the netfs whether an object on disk is still valid
911  * This function consults the netfs about the coherency state of an object.
912  * The caller must be holding a ref on cookie->n_active (held by
922 	if (!object->cookie->def->check_aux) {  in fscache_check_aux()
927 	result = object->cookie->def->check_aux(object->cookie->netfs_data,  in fscache_check_aux()
960 	struct fscache_cookie *cookie = object->cookie;  in _fscache_invalidate_object()
962 	_enter("{OBJ%x},%d", object->debug_id, event);  in _fscache_invalidate_object()
968 		ASSERT(radix_tree_empty(&object->cookie->stores));  in _fscache_invalidate_object()
969 		set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);  in _fscache_invalidate_object()
976 	clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);  in _fscache_invalidate_object()
979 	/* Now we have to wait for in-progress reads and writes */  in _fscache_invalidate_object()
984 	fscache_operation_init(cookie, op, object->cache->ops->invalidate_object,  in _fscache_invalidate_object()
986 	op->flags = FSCACHE_OP_ASYNC |  in _fscache_invalidate_object()
991 	spin_lock(&cookie->lock);  in _fscache_invalidate_object()
994 	spin_unlock(&cookie->lock);  in _fscache_invalidate_object()
998 	 * stored in the cache and thus we can reinstate the data-check-skip  in _fscache_invalidate_object()
1001 	set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in _fscache_invalidate_object()
1006 	if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))  in _fscache_invalidate_object()
1007 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);  in _fscache_invalidate_object()
1019 	spin_unlock(&cookie->lock);  in _fscache_invalidate_object()
1045 	object->cache->ops->update_object(object);  in fscache_update_aux_data()
1055 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_update_object()
1064  * fscache_object_retrying_stale - Note retrying stale object
1067  * Note that an object lookup found an on-disk object that was adjudged to be
1077  * fscache_object_mark_killed - Note that an object was killed
1087 	if (test_and_set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags)) {  in fscache_object_mark_killed()
1089 		       object->cache->identifier);  in fscache_object_mark_killed()
1120 			      &object->flags))  in fscache_object_dead()
1123 	WARN(true, "FS-Cache object redispatched after death");  in fscache_object_dead()