26 	iov_iter_xarray(&iter, ITER_DEST, &subreq->rreq->mapping->i_pages,  in netfs_clear_unread()
44 static void netfs_read_from_cache(struct netfs_io_request *rreq,  in netfs_read_from_cache()  argument
48 	struct netfs_cache_resources *cres = &rreq->cache_resources;  in netfs_read_from_cache()
52 	iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages,  in netfs_read_from_cache()
63 static void netfs_fill_with_zeroes(struct netfs_io_request *rreq,  in netfs_fill_with_zeroes()  argument
87 static void netfs_read_from_server(struct netfs_io_request *rreq,  in netfs_read_from_server()  argument
91 	rreq->netfs_ops->issue_read(subreq);  in netfs_read_from_server()
97 static void netfs_rreq_completed(struct netfs_io_request *rreq, bool was_async)  in netfs_rreq_completed()  argument
99 	trace_netfs_rreq(rreq, netfs_rreq_trace_done);  in netfs_rreq_completed()
100 	netfs_clear_subrequests(rreq, was_async);  in netfs_rreq_completed()
101 	netfs_put_request(rreq, was_async, netfs_rreq_trace_put_complete);  in netfs_rreq_completed()
110 static void netfs_rreq_unmark_after_write(struct netfs_io_request *rreq,  in netfs_rreq_unmark_after_write()  argument
120 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_unmark_after_write()
121 		XA_STATE(xas, &rreq->mapping->i_pages, subreq->start / PAGE_SIZE);  in netfs_rreq_unmark_after_write()
139 	netfs_rreq_completed(rreq, was_async);  in netfs_rreq_unmark_after_write()
146 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_rreq_copy_terminated()  local
150 		trace_netfs_failure(rreq, subreq, transferred_or_error,  in netfs_rreq_copy_terminated()
159 	if (atomic_dec_and_test(&rreq->nr_copy_ops))  in netfs_rreq_copy_terminated()
160 		netfs_rreq_unmark_after_write(rreq, was_async);  in netfs_rreq_copy_terminated()
169 static void netfs_rreq_do_write_to_cache(struct netfs_io_request *rreq)  in netfs_rreq_do_write_to_cache()  argument
171 	struct netfs_cache_resources *cres = &rreq->cache_resources;  in netfs_rreq_do_write_to_cache()
176 	trace_netfs_rreq(rreq, netfs_rreq_trace_copy);  in netfs_rreq_do_write_to_cache()
181 	atomic_inc(&rreq->nr_copy_ops);  in netfs_rreq_do_write_to_cache()
183 	list_for_each_entry_safe(subreq, p, &rreq->subrequests, rreq_link) {  in netfs_rreq_do_write_to_cache()
191 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_do_write_to_cache()
193 		while (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {  in netfs_rreq_do_write_to_cache()
204 					       rreq->i_size, true);  in netfs_rreq_do_write_to_cache()
206 			trace_netfs_failure(rreq, subreq, ret, netfs_fail_prepare_write);  in netfs_rreq_do_write_to_cache()
211 		iov_iter_xarray(&iter, ITER_SOURCE, &rreq->mapping->i_pages,  in netfs_rreq_do_write_to_cache()
214 		atomic_inc(&rreq->nr_copy_ops);  in netfs_rreq_do_write_to_cache()
223 	if (atomic_dec_and_test(&rreq->nr_copy_ops))  in netfs_rreq_do_write_to_cache()
224 		netfs_rreq_unmark_after_write(rreq, false);  in netfs_rreq_do_write_to_cache()
229 	struct netfs_io_request *rreq =  in netfs_rreq_write_to_cache_work()  local
232 	netfs_rreq_do_write_to_cache(rreq);  in netfs_rreq_write_to_cache_work()
235 static void netfs_rreq_write_to_cache(struct netfs_io_request *rreq)  in netfs_rreq_write_to_cache()  argument
237 	rreq->work.func = netfs_rreq_write_to_cache_work;  in netfs_rreq_write_to_cache()
238 	if (!queue_work(system_unbound_wq, &rreq->work))  in netfs_rreq_write_to_cache()
245 static void netfs_rreq_short_read(struct netfs_io_request *rreq,  in netfs_rreq_short_read()  argument
255 	atomic_inc(&rreq->nr_outstanding);  in netfs_rreq_short_read()
257 		netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_CLEAR);  in netfs_rreq_short_read()
259 		netfs_read_from_server(rreq, subreq);  in netfs_rreq_short_read()
266 static bool netfs_rreq_perform_resubmissions(struct netfs_io_request *rreq)  in netfs_rreq_perform_resubmissions()  argument
272 	trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);  in netfs_rreq_perform_resubmissions()
277 	atomic_inc(&rreq->nr_outstanding);  in netfs_rreq_perform_resubmissions()
279 	__clear_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);  in netfs_rreq_perform_resubmissions()
280 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_perform_resubmissions()
289 			atomic_inc(&rreq->nr_outstanding);  in netfs_rreq_perform_resubmissions()
290 			netfs_read_from_server(rreq, subreq);  in netfs_rreq_perform_resubmissions()
292 			netfs_rreq_short_read(rreq, subreq);  in netfs_rreq_perform_resubmissions()
297 	if (atomic_dec_and_test(&rreq->nr_outstanding))  in netfs_rreq_perform_resubmissions()
300 	wake_up_var(&rreq->nr_outstanding);  in netfs_rreq_perform_resubmissions()
307 static void netfs_rreq_is_still_valid(struct netfs_io_request *rreq)  in netfs_rreq_is_still_valid()  argument
311 	if (!rreq->netfs_ops->is_still_valid ||  in netfs_rreq_is_still_valid()
312 	    rreq->netfs_ops->is_still_valid(rreq))  in netfs_rreq_is_still_valid()
315 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_is_still_valid()
318 			__set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);  in netfs_rreq_is_still_valid()
329 static void netfs_rreq_assess(struct netfs_io_request *rreq, bool was_async)  in netfs_rreq_assess()  argument
331 	trace_netfs_rreq(rreq, netfs_rreq_trace_assess);  in netfs_rreq_assess()
334 	netfs_rreq_is_still_valid(rreq);  in netfs_rreq_assess()
336 	if (!test_bit(NETFS_RREQ_FAILED, &rreq->flags) &&  in netfs_rreq_assess()
337 	    test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags)) {  in netfs_rreq_assess()
338 		if (netfs_rreq_perform_resubmissions(rreq))  in netfs_rreq_assess()
343 	netfs_rreq_unlock_folios(rreq);  in netfs_rreq_assess()
345 	clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags);  in netfs_rreq_assess()
346 	wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS);  in netfs_rreq_assess()
348 	if (test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags))  in netfs_rreq_assess()
349 		return netfs_rreq_write_to_cache(rreq);  in netfs_rreq_assess()
351 	netfs_rreq_completed(rreq, was_async);  in netfs_rreq_assess()
356 	struct netfs_io_request *rreq =  in netfs_rreq_work()  local
358 	netfs_rreq_assess(rreq, false);  in netfs_rreq_work()
365 static void netfs_rreq_terminated(struct netfs_io_request *rreq,  in netfs_rreq_terminated()  argument
368 	if (test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags) &&  in netfs_rreq_terminated()
370 		if (!queue_work(system_unbound_wq, &rreq->work))  in netfs_rreq_terminated()
373 		netfs_rreq_assess(rreq, was_async);  in netfs_rreq_terminated()
399 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_subreq_terminated()  local
419 		trace_netfs_failure(rreq, subreq, transferred_or_error,  in netfs_subreq_terminated()
426 		 rreq->debug_id, subreq->debug_index,  in netfs_subreq_terminated()
438 		set_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags);  in netfs_subreq_terminated()
444 	u = atomic_dec_return(&rreq->nr_outstanding);  in netfs_subreq_terminated()
446 		netfs_rreq_terminated(rreq, was_async);  in netfs_subreq_terminated()
448 		wake_up_var(&rreq->nr_outstanding);  in netfs_subreq_terminated()
470 	set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);  in netfs_subreq_terminated()
476 		set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);  in netfs_subreq_terminated()
479 		set_bit(NETFS_RREQ_FAILED, &rreq->flags);  in netfs_subreq_terminated()
480 		rreq->error = subreq->error;  in netfs_subreq_terminated()
489 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_cache_prepare_read()  local
490 	struct netfs_cache_resources *cres = &rreq->cache_resources;  in netfs_cache_prepare_read()
494 	if (subreq->start >= rreq->i_size)  in netfs_cache_prepare_read()
503 netfs_rreq_prepare_read(struct netfs_io_request *rreq,  in netfs_rreq_prepare_read()  argument
508 	_enter("%llx-%llx,%llx", subreq->start, subreq->start + subreq->len, rreq->i_size);  in netfs_rreq_prepare_read()
510 	source = netfs_cache_prepare_read(subreq, rreq->i_size);  in netfs_rreq_prepare_read()
521 		if (subreq->len > rreq->i_size - subreq->start)  in netfs_rreq_prepare_read()
522 			subreq->len = rreq->i_size - subreq->start;  in netfs_rreq_prepare_read()
524 		if (rreq->netfs_ops->clamp_length &&  in netfs_rreq_prepare_read()
525 		    !rreq->netfs_ops->clamp_length(subreq)) {  in netfs_rreq_prepare_read()
543 static bool netfs_rreq_submit_slice(struct netfs_io_request *rreq,  in netfs_rreq_submit_slice()  argument
549 	subreq = netfs_alloc_subrequest(rreq);  in netfs_rreq_submit_slice()
554 	subreq->start		= rreq->start + rreq->submitted;  in netfs_rreq_submit_slice()
555 	subreq->len		= rreq->len   - rreq->submitted;  in netfs_rreq_submit_slice()
557 	_debug("slice %llx,%zx,%zx", subreq->start, subreq->len, rreq->submitted);  in netfs_rreq_submit_slice()
558 	list_add_tail(&subreq->rreq_link, &rreq->subrequests);  in netfs_rreq_submit_slice()
568 	source = netfs_rreq_prepare_read(rreq, subreq);  in netfs_rreq_submit_slice()
572 	atomic_inc(&rreq->nr_outstanding);  in netfs_rreq_submit_slice()
574 	rreq->submitted += subreq->len;  in netfs_rreq_submit_slice()
579 		netfs_fill_with_zeroes(rreq, subreq);  in netfs_rreq_submit_slice()
582 		netfs_read_from_server(rreq, subreq);  in netfs_rreq_submit_slice()
585 		netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_IGNORE);  in netfs_rreq_submit_slice()
594 	rreq->error = subreq->error;  in netfs_rreq_submit_slice()
604 int netfs_begin_read(struct netfs_io_request *rreq, bool sync)  in netfs_begin_read()  argument
610 	       rreq->debug_id, rreq->start, rreq->start + rreq->len - 1);  in netfs_begin_read()
612 	if (rreq->len == 0) {  in netfs_begin_read()
613 		pr_err("Zero-sized read [R=%x]\n", rreq->debug_id);  in netfs_begin_read()
614 		netfs_put_request(rreq, false, netfs_rreq_trace_put_zero_len);  in netfs_begin_read()
618 	INIT_WORK(&rreq->work, netfs_rreq_work);  in netfs_begin_read()
621 		netfs_get_request(rreq, netfs_rreq_trace_get_hold);  in netfs_begin_read()
626 	atomic_set(&rreq->nr_outstanding, 1);  in netfs_begin_read()
628 		if (!netfs_rreq_submit_slice(rreq, &debug_index))  in netfs_begin_read()
631 	} while (rreq->submitted < rreq->len);  in netfs_begin_read()
639 			wait_var_event(&rreq->nr_outstanding,  in netfs_begin_read()
640 				       atomic_read(&rreq->nr_outstanding) == 1);  in netfs_begin_read()
641 			netfs_rreq_assess(rreq, false);  in netfs_begin_read()
642 			if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags))  in netfs_begin_read()
647 		ret = rreq->error;  in netfs_begin_read()
648 		if (ret == 0 && rreq->submitted < rreq->len) {  in netfs_begin_read()
649 			trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);  in netfs_begin_read()
652 		netfs_put_request(rreq, false, netfs_rreq_trace_put_hold);  in netfs_begin_read()
655 		if (atomic_dec_and_test(&rreq->nr_outstanding))  in netfs_begin_read()
656 			netfs_rreq_assess(rreq, false);  in netfs_begin_read()