22 static void netfs_clear_unread(struct netfs_io_subrequest *subreq)  in netfs_clear_unread()  argument
26 	iov_iter_xarray(&iter, ITER_DEST, &subreq->rreq->mapping->i_pages,  in netfs_clear_unread()
27 			subreq->start + subreq->transferred,  in netfs_clear_unread()
28 			subreq->len   - subreq->transferred);  in netfs_clear_unread()
35 	struct netfs_io_subrequest *subreq = priv;  in netfs_cache_read_terminated()  local
37 	netfs_subreq_terminated(subreq, transferred_or_error, was_async);  in netfs_cache_read_terminated()
45 				  struct netfs_io_subrequest *subreq,  in netfs_read_from_cache()  argument
53 			subreq->start + subreq->transferred,  in netfs_read_from_cache()
54 			subreq->len   - subreq->transferred);  in netfs_read_from_cache()
56 	cres->ops->read(cres, subreq->start, &iter, read_hole,  in netfs_read_from_cache()
57 			netfs_cache_read_terminated, subreq);  in netfs_read_from_cache()
64 				   struct netfs_io_subrequest *subreq)  in netfs_fill_with_zeroes()  argument
67 	__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);  in netfs_fill_with_zeroes()
68 	netfs_subreq_terminated(subreq, 0, false);  in netfs_fill_with_zeroes()
88 				   struct netfs_io_subrequest *subreq)  in netfs_read_from_server()  argument
91 	rreq->netfs_ops->issue_read(subreq);  in netfs_read_from_server()
113 	struct netfs_io_subrequest *subreq;  in netfs_rreq_unmark_after_write()  local
120 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_unmark_after_write()
121 		XA_STATE(xas, &rreq->mapping->i_pages, subreq->start / PAGE_SIZE);  in netfs_rreq_unmark_after_write()
123 		xas_for_each(&xas, folio, (subreq->start + subreq->len - 1) / PAGE_SIZE) {  in netfs_rreq_unmark_after_write()
145 	struct netfs_io_subrequest *subreq = priv;  in netfs_rreq_copy_terminated()  local
146 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_rreq_copy_terminated()
150 		trace_netfs_failure(rreq, subreq, transferred_or_error,  in netfs_rreq_copy_terminated()
156 	trace_netfs_sreq(subreq, netfs_sreq_trace_write_term);  in netfs_rreq_copy_terminated()
162 	netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated);  in netfs_rreq_copy_terminated()
172 	struct netfs_io_subrequest *subreq, *next, *p;  in netfs_rreq_do_write_to_cache()  local
183 	list_for_each_entry_safe(subreq, p, &rreq->subrequests, rreq_link) {  in netfs_rreq_do_write_to_cache()
184 		if (!test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) {  in netfs_rreq_do_write_to_cache()
185 			list_del_init(&subreq->rreq_link);  in netfs_rreq_do_write_to_cache()
186 			netfs_put_subrequest(subreq, false,  in netfs_rreq_do_write_to_cache()
191 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_do_write_to_cache()
193 		while (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {  in netfs_rreq_do_write_to_cache()
194 			next = list_next_entry(subreq, rreq_link);  in netfs_rreq_do_write_to_cache()
195 			if (next->start != subreq->start + subreq->len)  in netfs_rreq_do_write_to_cache()
197 			subreq->len += next->len;  in netfs_rreq_do_write_to_cache()
203 		ret = cres->ops->prepare_write(cres, &subreq->start, &subreq->len,  in netfs_rreq_do_write_to_cache()
206 			trace_netfs_failure(rreq, subreq, ret, netfs_fail_prepare_write);  in netfs_rreq_do_write_to_cache()
207 			trace_netfs_sreq(subreq, netfs_sreq_trace_write_skip);  in netfs_rreq_do_write_to_cache()
212 				subreq->start, subreq->len);  in netfs_rreq_do_write_to_cache()
216 		netfs_get_subrequest(subreq, netfs_sreq_trace_get_copy_to_cache);  in netfs_rreq_do_write_to_cache()
217 		trace_netfs_sreq(subreq, netfs_sreq_trace_write);  in netfs_rreq_do_write_to_cache()
218 		cres->ops->write(cres, subreq->start, &iter,  in netfs_rreq_do_write_to_cache()
219 				 netfs_rreq_copy_terminated, subreq);  in netfs_rreq_do_write_to_cache()
246 				  struct netfs_io_subrequest *subreq)  in netfs_rreq_short_read()  argument
248 	__clear_bit(NETFS_SREQ_SHORT_IO, &subreq->flags);  in netfs_rreq_short_read()
249 	__set_bit(NETFS_SREQ_SEEK_DATA_READ, &subreq->flags);  in netfs_rreq_short_read()
252 	trace_netfs_sreq(subreq, netfs_sreq_trace_resubmit_short);  in netfs_rreq_short_read()
254 	netfs_get_subrequest(subreq, netfs_sreq_trace_get_short_read);  in netfs_rreq_short_read()
256 	if (subreq->source == NETFS_READ_FROM_CACHE)  in netfs_rreq_short_read()
257 		netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_CLEAR);  in netfs_rreq_short_read()
259 		netfs_read_from_server(rreq, subreq);  in netfs_rreq_short_read()
268 	struct netfs_io_subrequest *subreq;  in netfs_rreq_perform_resubmissions()  local
280 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_perform_resubmissions()
281 		if (subreq->error) {  in netfs_rreq_perform_resubmissions()
282 			if (subreq->source != NETFS_READ_FROM_CACHE)  in netfs_rreq_perform_resubmissions()
284 			subreq->source = NETFS_DOWNLOAD_FROM_SERVER;  in netfs_rreq_perform_resubmissions()
285 			subreq->error = 0;  in netfs_rreq_perform_resubmissions()
287 			trace_netfs_sreq(subreq, netfs_sreq_trace_download_instead);  in netfs_rreq_perform_resubmissions()
288 			netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);  in netfs_rreq_perform_resubmissions()
290 			netfs_read_from_server(rreq, subreq);  in netfs_rreq_perform_resubmissions()
291 		} else if (test_bit(NETFS_SREQ_SHORT_IO, &subreq->flags)) {  in netfs_rreq_perform_resubmissions()
292 			netfs_rreq_short_read(rreq, subreq);  in netfs_rreq_perform_resubmissions()
309 	struct netfs_io_subrequest *subreq;  in netfs_rreq_is_still_valid()  local
315 	list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {  in netfs_rreq_is_still_valid()
316 		if (subreq->source == NETFS_READ_FROM_CACHE) {  in netfs_rreq_is_still_valid()
317 			subreq->error = -ESTALE;  in netfs_rreq_is_still_valid()
395 void netfs_subreq_terminated(struct netfs_io_subrequest *subreq,  in netfs_subreq_terminated()  argument
399 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_subreq_terminated()
403 	       subreq->debug_index, subreq->start, subreq->flags,  in netfs_subreq_terminated()
406 	switch (subreq->source) {  in netfs_subreq_terminated()
418 		subreq->error = transferred_or_error;  in netfs_subreq_terminated()
419 		trace_netfs_failure(rreq, subreq, transferred_or_error,  in netfs_subreq_terminated()
424 	if (WARN(transferred_or_error > subreq->len - subreq->transferred,  in netfs_subreq_terminated()
426 		 rreq->debug_id, subreq->debug_index,  in netfs_subreq_terminated()
427 		 transferred_or_error, subreq->len, subreq->transferred))  in netfs_subreq_terminated()
428 		transferred_or_error = subreq->len - subreq->transferred;  in netfs_subreq_terminated()
430 	subreq->error = 0;  in netfs_subreq_terminated()
431 	subreq->transferred += transferred_or_error;  in netfs_subreq_terminated()
432 	if (subreq->transferred < subreq->len)  in netfs_subreq_terminated()
436 	__clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);  in netfs_subreq_terminated()
437 	if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags))  in netfs_subreq_terminated()
441 	trace_netfs_sreq(subreq, netfs_sreq_trace_terminated);  in netfs_subreq_terminated()
450 	netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated);  in netfs_subreq_terminated()
454 	if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) {  in netfs_subreq_terminated()
455 		netfs_clear_unread(subreq);  in netfs_subreq_terminated()
456 		subreq->transferred = subreq->len;  in netfs_subreq_terminated()
461 		if (__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) {  in netfs_subreq_terminated()
462 			subreq->error = -ENODATA;  in netfs_subreq_terminated()
466 		__clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);  in netfs_subreq_terminated()
469 	__set_bit(NETFS_SREQ_SHORT_IO, &subreq->flags);  in netfs_subreq_terminated()
474 	if (subreq->source == NETFS_READ_FROM_CACHE) {  in netfs_subreq_terminated()
480 		rreq->error = subreq->error;  in netfs_subreq_terminated()
486 static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_subrequest *subreq,  in netfs_cache_prepare_read()  argument
489 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_cache_prepare_read()
493 		return cres->ops->prepare_read(subreq, i_size);  in netfs_cache_prepare_read()
494 	if (subreq->start >= rreq->i_size)  in netfs_cache_prepare_read()
504 			struct netfs_io_subrequest *subreq)  in netfs_rreq_prepare_read()  argument
508 	_enter("%llx-%llx,%llx", subreq->start, subreq->start + subreq->len, rreq->i_size);  in netfs_rreq_prepare_read()
510 	source = netfs_cache_prepare_read(subreq, rreq->i_size);  in netfs_rreq_prepare_read()
521 		if (subreq->len > rreq->i_size - subreq->start)  in netfs_rreq_prepare_read()
522 			subreq->len = rreq->i_size - subreq->start;  in netfs_rreq_prepare_read()
525 		    !rreq->netfs_ops->clamp_length(subreq)) {  in netfs_rreq_prepare_read()
531 	if (WARN_ON(subreq->len == 0))  in netfs_rreq_prepare_read()
535 	subreq->source = source;  in netfs_rreq_prepare_read()
536 	trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);  in netfs_rreq_prepare_read()
546 	struct netfs_io_subrequest *subreq;  in netfs_rreq_submit_slice()  local
549 	subreq = netfs_alloc_subrequest(rreq);  in netfs_rreq_submit_slice()
550 	if (!subreq)  in netfs_rreq_submit_slice()
553 	subreq->debug_index	= (*_debug_index)++;  in netfs_rreq_submit_slice()
554 	subreq->start		= rreq->start + rreq->submitted;  in netfs_rreq_submit_slice()
555 	subreq->len		= rreq->len   - rreq->submitted;  in netfs_rreq_submit_slice()
557 	_debug("slice %llx,%zx,%zx", subreq->start, subreq->len, rreq->submitted);  in netfs_rreq_submit_slice()
558 	list_add_tail(&subreq->rreq_link, &rreq->subrequests);  in netfs_rreq_submit_slice()
568 	source = netfs_rreq_prepare_read(rreq, subreq);  in netfs_rreq_submit_slice()
574 	rreq->submitted += subreq->len;  in netfs_rreq_submit_slice()
576 	trace_netfs_sreq(subreq, netfs_sreq_trace_submit);  in netfs_rreq_submit_slice()
579 		netfs_fill_with_zeroes(rreq, subreq);  in netfs_rreq_submit_slice()
582 		netfs_read_from_server(rreq, subreq);  in netfs_rreq_submit_slice()
585 		netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_IGNORE);  in netfs_rreq_submit_slice()
594 	rreq->error = subreq->error;  in netfs_rreq_submit_slice()
595 	netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_failed);  in netfs_rreq_submit_slice()