1 // SPDX-License-Identifier: GPL-2.0-only
37 	if (desc->pg_ops->pg_get_mirror)  in nfs_pgio_get_mirror()
38 		return desc->pg_ops->pg_get_mirror(desc, idx);  in nfs_pgio_get_mirror()
39 	return &desc->pg_mirrors[0];  in nfs_pgio_get_mirror()
45 	return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx);  in nfs_pgio_current_mirror()
52 	if (desc->pg_ops->pg_set_mirror)  in nfs_pgio_set_current_mirror()
53 		return desc->pg_ops->pg_set_mirror(desc, idx);  in nfs_pgio_set_current_mirror()
54 	return desc->pg_mirror_idx;  in nfs_pgio_set_current_mirror()
61 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pgheader_init()  local
64 	hdr->req = nfs_list_entry(mirror->pg_list.next);  in nfs_pgheader_init()
65 	hdr->inode = desc->pg_inode;  in nfs_pgheader_init()
66 	hdr->cred = nfs_req_openctx(hdr->req)->cred;  in nfs_pgheader_init()
67 	hdr->io_start = req_offset(hdr->req);  in nfs_pgheader_init()
68 	hdr->good_bytes = mirror->pg_count;  in nfs_pgheader_init()
69 	hdr->io_completion = desc->pg_io_completion;  in nfs_pgheader_init()
70 	hdr->dreq = desc->pg_dreq;  in nfs_pgheader_init()
71 	hdr->release = release;  in nfs_pgheader_init()
72 	hdr->completion_ops = desc->pg_completion_ops;  in nfs_pgheader_init()
73 	if (hdr->completion_ops->init_hdr)  in nfs_pgheader_init()
74 		hdr->completion_ops->init_hdr(hdr);  in nfs_pgheader_init()
76 	hdr->pgio_mirror_idx = desc->pg_mirror_idx;  in nfs_pgheader_init()
82 	unsigned int new = pos - hdr->io_start;  in nfs_set_pgio_error()
85 	if (hdr->good_bytes > new) {  in nfs_set_pgio_error()
86 		hdr->good_bytes = new;  in nfs_set_pgio_error()
87 		clear_bit(NFS_IOHDR_EOF, &hdr->flags);  in nfs_set_pgio_error()
88 		if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags))  in nfs_set_pgio_error()
89 			hdr->error = error;  in nfs_set_pgio_error()
98 		INIT_LIST_HEAD(&p->wb_list);  in nfs_page_alloc()
109  * nfs_iocounter_wait - wait for i/o to complete
112  * returns -ERESTARTSYS if interrupted by a fatal signal.
118 	return wait_var_event_killable(&l_ctx->io_count,  in nfs_iocounter_wait()
119 				       !atomic_read(&l_ctx->io_count));  in nfs_iocounter_wait()
123  * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
134 	struct inode *inode = d_inode(l_ctx->open_context->dentry);  in nfs_async_iocounter_wait()
137 	if (atomic_read(&l_ctx->io_count) > 0) {  in nfs_async_iocounter_wait()
138 		rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL);  in nfs_async_iocounter_wait()
142 	if (atomic_read(&l_ctx->io_count) == 0) {  in nfs_async_iocounter_wait()
143 		rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task);  in nfs_async_iocounter_wait()
152  * nfs_page_lock_head_request - page lock the head of the page group
158 	struct nfs_page *head = req->wb_head;  in nfs_page_group_lock_head()
166 		kref_get(&head->wb_kref);  in nfs_page_group_lock_head()
171  * nfs_unroll_locks -  unlock all newly locked reqs and wait on @req
184 	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {  in nfs_unroll_locks()
185 		if (!kref_read(&tmp->wb_kref))  in nfs_unroll_locks()
192  * nfs_page_group_lock_subreq -  try to lock a subrequest
205 	if (!kref_get_unless_zero(&subreq->wb_kref))  in nfs_page_group_lock_subreq()
222  * nfs_page_group_lock_subrequests -  try to lock the subrequests
237 	for (subreq = head->wb_this_page; subreq != head;  in nfs_page_group_lock_subrequests()
238 			subreq = subreq->wb_this_page) {  in nfs_page_group_lock_subrequests()
248  * nfs_page_set_headlock - set the request PG_HEADLOCK
251  * this lock must be held when modifying req->wb_head
258 	if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags))  in nfs_page_set_headlock()
261 	set_bit(PG_CONTENDED1, &req->wb_flags);  in nfs_page_set_headlock()
263 	return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK,  in nfs_page_set_headlock()
268  * nfs_page_clear_headlock - clear the request PG_HEADLOCK
274 	clear_bit_unlock(PG_HEADLOCK, &req->wb_flags);  in nfs_page_clear_headlock()
276 	if (!test_bit(PG_CONTENDED1, &req->wb_flags))  in nfs_page_clear_headlock()
278 	wake_up_bit(&req->wb_flags, PG_HEADLOCK);  in nfs_page_clear_headlock()
282  * nfs_page_group_lock - lock the head of the page group
296 	if (ret || req->wb_head == req)  in nfs_page_group_lock()
298 	return nfs_page_set_headlock(req->wb_head);  in nfs_page_group_lock()
302  * nfs_page_group_unlock - unlock the head of the page group
308 	if (req != req->wb_head)  in nfs_page_group_unlock()
309 		nfs_page_clear_headlock(req->wb_head);  in nfs_page_group_unlock()
321 	struct nfs_page *head = req->wb_head;  in nfs_page_group_sync_on_bit_locked()
324 	WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));  in nfs_page_group_sync_on_bit_locked()
325 	WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));  in nfs_page_group_sync_on_bit_locked()
327 	tmp = req->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
329 		if (!test_bit(bit, &tmp->wb_flags))  in nfs_page_group_sync_on_bit_locked()
331 		tmp = tmp->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
337 		clear_bit(bit, &tmp->wb_flags);  in nfs_page_group_sync_on_bit_locked()
338 		tmp = tmp->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
345  * nfs_page_group_sync_on_bit - set bit on current request, but only
347  * @req - request in page group
348  * @bit - PG_* bit that is used to sync page group
362  * nfs_page_group_init - Initialize the page group linkage for @req
363  * @req - a new nfs request
364  * @prev - the previous request in page group, or NULL if @req is the first
375 		req->wb_head = req;  in nfs_page_group_init()
376 		req->wb_this_page = req;  in nfs_page_group_init()
379 		WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);  in nfs_page_group_init()
380 		WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));  in nfs_page_group_init()
381 		req->wb_head = prev->wb_head;  in nfs_page_group_init()
382 		req->wb_this_page = prev->wb_this_page;  in nfs_page_group_init()
383 		prev->wb_this_page = req;  in nfs_page_group_init()
387 		kref_get(&req->wb_head->wb_kref);  in nfs_page_group_init()
392 		if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {  in nfs_page_group_init()
393 			inode = page_file_mapping(req->wb_page)->host;  in nfs_page_group_init()
394 			set_bit(PG_INODE_REF, &req->wb_flags);  in nfs_page_group_init()
395 			kref_get(&req->wb_kref);  in nfs_page_group_init()
396 			atomic_long_inc(&NFS_I(inode)->nrequests);  in nfs_page_group_init()
402  * nfs_page_group_destroy - sync the destruction of page groups
403  * @req - request that no longer needs the page group
412 	struct nfs_page *head = req->wb_head;  in nfs_page_group_destroy()
420 		next = tmp->wb_this_page;  in nfs_page_group_destroy()
422 		tmp->wb_this_page = tmp;  in nfs_page_group_destroy()
423 		tmp->wb_head = tmp;  in nfs_page_group_destroy()
439 	struct nfs_open_context *ctx = l_ctx->open_context;  in __nfs_create_request()
441 	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))  in __nfs_create_request()
442 		return ERR_PTR(-EBADF);  in __nfs_create_request()
446 		return ERR_PTR(-ENOMEM);  in __nfs_create_request()
448 	req->wb_lock_context = l_ctx;  in __nfs_create_request()
449 	refcount_inc(&l_ctx->count);  in __nfs_create_request()
450 	atomic_inc(&l_ctx->io_count);  in __nfs_create_request()
453 	 * long write-back delay. This will be adjusted in  in __nfs_create_request()
455 	req->wb_page    = page;  in __nfs_create_request()
457 		req->wb_index = page_index(page);  in __nfs_create_request()
460 	req->wb_offset  = offset;  in __nfs_create_request()
461 	req->wb_pgbase	= pgbase;  in __nfs_create_request()
462 	req->wb_bytes   = count;  in __nfs_create_request()
463 	kref_init(&req->wb_kref);  in __nfs_create_request()
464 	req->wb_nio = 0;  in __nfs_create_request()
469  * nfs_create_request - Create an NFS read/write request.
504 	ret = __nfs_create_request(req->wb_lock_context, req->wb_page,  in nfs_create_subreq()
508 		for (last = req->wb_head;  in nfs_create_subreq()
509 		     last->wb_this_page != req->wb_head;  in nfs_create_subreq()
510 		     last = last->wb_this_page)  in nfs_create_subreq()
514 		ret->wb_index = req->wb_index;  in nfs_create_subreq()
516 		ret->wb_nio = req->wb_nio;  in nfs_create_subreq()
522  * nfs_unlock_request - Unlock request and wake up sleepers.
527 	clear_bit_unlock(PG_BUSY, &req->wb_flags);  in nfs_unlock_request()
529 	if (!test_bit(PG_CONTENDED2, &req->wb_flags))  in nfs_unlock_request()
531 	wake_up_bit(&req->wb_flags, PG_BUSY);  in nfs_unlock_request()
535  * nfs_unlock_and_release_request - Unlock request and release the nfs_page
545  * nfs_clear_request - Free up all resources allocated to the request
553 	struct page *page = req->wb_page;  in nfs_clear_request()
554 	struct nfs_lock_context *l_ctx = req->wb_lock_context;  in nfs_clear_request()
559 		req->wb_page = NULL;  in nfs_clear_request()
562 		if (atomic_dec_and_test(&l_ctx->io_count)) {  in nfs_clear_request()
563 			wake_up_var(&l_ctx->io_count);  in nfs_clear_request()
564 			ctx = l_ctx->open_context;  in nfs_clear_request()
565 			if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags))  in nfs_clear_request()
566 				rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq);  in nfs_clear_request()
569 		req->wb_lock_context = NULL;  in nfs_clear_request()
574  * nfs_free_request - Release the count on an NFS read/write request
581 	WARN_ON_ONCE(req->wb_this_page != req);  in nfs_free_request()
584 	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));  in nfs_free_request()
585 	WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));  in nfs_free_request()
586 	WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));  in nfs_free_request()
587 	WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));  in nfs_free_request()
588 	WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));  in nfs_free_request()
597 	kref_put(&req->wb_kref, nfs_page_group_destroy);  in nfs_release_request()
602  * nfs_wait_on_request - Wait for a request to complete.
611 	if (!test_bit(PG_BUSY, &req->wb_flags))  in nfs_wait_on_request()
613 	set_bit(PG_CONTENDED2, &req->wb_flags);  in nfs_wait_on_request()
615 	return wait_on_bit_io(&req->wb_flags, PG_BUSY,  in nfs_wait_on_request()
621  * nfs_generic_pg_test - determine if requests can be coalesced
632 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pg_test()  local
635 	if (mirror->pg_count > mirror->pg_bsize) {  in nfs_generic_pg_test()
645 	if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *  in nfs_generic_pg_test()
649 	return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);  in nfs_generic_pg_test()
655 	struct nfs_pgio_header *hdr = ops->rw_alloc_header();  in nfs_pgio_header_alloc()
658 		INIT_LIST_HEAD(&hdr->pages);  in nfs_pgio_header_alloc()
659 		hdr->rw_ops = ops;  in nfs_pgio_header_alloc()
666  * nfs_pgio_data_destroy - make @hdr suitable for reuse
675 	if (hdr->args.context)  in nfs_pgio_data_destroy()
676 		put_nfs_open_context(hdr->args.context);  in nfs_pgio_data_destroy()
677 	if (hdr->page_array.pagevec != hdr->page_array.page_array)  in nfs_pgio_data_destroy()
678 		kfree(hdr->page_array.pagevec);  in nfs_pgio_data_destroy()
682  * nfs_pgio_header_free - Free a read or write header
688 	hdr->rw_ops->rw_free_header(hdr);  in nfs_pgio_header_free()
693  * nfs_pgio_rpcsetup - Set up arguments for a pageio call
696  * @how: How to commit data (writes only)
703 	struct nfs_page *req = hdr->req;  in nfs_pgio_rpcsetup()
706 	 * NB: take care not to mess about with hdr->commit et al. */  in nfs_pgio_rpcsetup()
708 	hdr->args.fh     = NFS_FH(hdr->inode);  in nfs_pgio_rpcsetup()
709 	hdr->args.offset = req_offset(req);  in nfs_pgio_rpcsetup()
711 	hdr->mds_offset = hdr->args.offset;  in nfs_pgio_rpcsetup()
712 	hdr->args.pgbase = req->wb_pgbase;  in nfs_pgio_rpcsetup()
713 	hdr->args.pages  = hdr->page_array.pagevec;  in nfs_pgio_rpcsetup()
714 	hdr->args.count  = count;  in nfs_pgio_rpcsetup()
715 	hdr->args.context = get_nfs_open_context(nfs_req_openctx(req));  in nfs_pgio_rpcsetup()
716 	hdr->args.lock_context = req->wb_lock_context;  in nfs_pgio_rpcsetup()
717 	hdr->args.stable  = NFS_UNSTABLE;  in nfs_pgio_rpcsetup()
726 		hdr->args.stable = NFS_FILE_SYNC;  in nfs_pgio_rpcsetup()
729 	hdr->res.fattr   = &hdr->fattr;  in nfs_pgio_rpcsetup()
730 	hdr->res.count   = 0;  in nfs_pgio_rpcsetup()
731 	hdr->res.eof     = 0;  in nfs_pgio_rpcsetup()
732 	hdr->res.verf    = &hdr->verf;  in nfs_pgio_rpcsetup()
733 	nfs_fattr_init(&hdr->fattr);  in nfs_pgio_rpcsetup()
737  * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
745 	err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr);  in nfs_pgio_prepare()
756 		.rpc_argp = &hdr->args,  in nfs_initiate_pgio()
757 		.rpc_resp = &hdr->res,  in nfs_initiate_pgio()
762 		.task = &hdr->task,  in nfs_initiate_pgio()
770 	if (nfs_server_capable(hdr->inode, NFS_CAP_MOVEABLE))  in nfs_initiate_pgio()
773 	hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);  in nfs_initiate_pgio()
777 		hdr->inode->i_sb->s_id,  in nfs_initiate_pgio()
778 		(unsigned long long)NFS_FILEID(hdr->inode),  in nfs_initiate_pgio()
779 		hdr->args.count,  in nfs_initiate_pgio()
780 		(unsigned long long)hdr->args.offset);  in nfs_initiate_pgio()
791  * nfs_pgio_error - Clean up from a pageio error
796 	set_bit(NFS_IOHDR_REDO, &hdr->flags);  in nfs_pgio_error()
797 	hdr->completion_ops->completion(hdr);  in nfs_pgio_error()
801  * nfs_pgio_release - Release pageio data
807 	hdr->completion_ops->completion(hdr);  in nfs_pgio_release()
810 static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,  in nfs_pageio_mirror_init()  argument
813 	INIT_LIST_HEAD(&mirror->pg_list);  in nfs_pageio_mirror_init()
814 	mirror->pg_bytes_written = 0;  in nfs_pageio_mirror_init()
815 	mirror->pg_count = 0;  in nfs_pageio_mirror_init()
816 	mirror->pg_bsize = bsize;  in nfs_pageio_mirror_init()
817 	mirror->pg_base = 0;  in nfs_pageio_mirror_init()
818 	mirror->pg_recoalesce = 0;  in nfs_pageio_mirror_init()
822  * nfs_pageio_init - initialise a page io descriptor
839 	desc->pg_moreio = 0;  in nfs_pageio_init()
840 	desc->pg_inode = inode;  in nfs_pageio_init()
841 	desc->pg_ops = pg_ops;  in nfs_pageio_init()
842 	desc->pg_completion_ops = compl_ops;  in nfs_pageio_init()
843 	desc->pg_rw_ops = rw_ops;  in nfs_pageio_init()
844 	desc->pg_ioflags = io_flags;  in nfs_pageio_init()
845 	desc->pg_error = 0;  in nfs_pageio_init()
846 	desc->pg_lseg = NULL;  in nfs_pageio_init()
847 	desc->pg_io_completion = NULL;  in nfs_pageio_init()
848 	desc->pg_dreq = NULL;  in nfs_pageio_init()
849 	desc->pg_bsize = bsize;  in nfs_pageio_init()
851 	desc->pg_mirror_count = 1;  in nfs_pageio_init()
852 	desc->pg_mirror_idx = 0;  in nfs_pageio_init()
854 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_init()
855 	desc->pg_mirrors = desc->pg_mirrors_static;  in nfs_pageio_init()
856 	nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);  in nfs_pageio_init()
857 	desc->pg_maxretrans = 0;  in nfs_pageio_init()
861  * nfs_pgio_result - Basic pageio error handling
868 	struct inode *inode = hdr->inode;  in nfs_pgio_result()
870 	if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)  in nfs_pgio_result()
872 	if (task->tk_status < 0)  in nfs_pgio_result()
873 		nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset);  in nfs_pgio_result()
875 		hdr->rw_ops->rw_result(task, hdr);  in nfs_pgio_result()
889 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pgio()  local
894 	struct list_head *head = &mirror->pg_list;  in nfs_generic_pgio()
896 	struct nfs_page_array *pg_array = &hdr->page_array;  in nfs_generic_pgio()
900 	pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);  in nfs_generic_pgio()
901 	pg_array->npages = pagecount;  in nfs_generic_pgio()
903 	if (pagecount <= ARRAY_SIZE(pg_array->page_array))  in nfs_generic_pgio()
904 		pg_array->pagevec = pg_array->page_array;  in nfs_generic_pgio()
906 		pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags);  in nfs_generic_pgio()
907 		if (!pg_array->pagevec) {  in nfs_generic_pgio()
908 			pg_array->npages = 0;  in nfs_generic_pgio()
910 			desc->pg_error = -ENOMEM;  in nfs_generic_pgio()
911 			return desc->pg_error;  in nfs_generic_pgio()
915 	nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);  in nfs_generic_pgio()
916 	pages = hdr->page_array.pagevec;  in nfs_generic_pgio()
920 		req = nfs_list_entry(head->next);  in nfs_generic_pgio()
921 		nfs_list_move_request(req, &hdr->pages);  in nfs_generic_pgio()
923 		if (!last_page || last_page != req->wb_page) {  in nfs_generic_pgio()
927 			*pages++ = last_page = req->wb_page;  in nfs_generic_pgio()
932 		desc->pg_error = -EINVAL;  in nfs_generic_pgio()
933 		return desc->pg_error;  in nfs_generic_pgio()
936 	if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&  in nfs_generic_pgio()
937 	    (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))  in nfs_generic_pgio()
938 		desc->pg_ioflags &= ~FLUSH_COND_STABLE;  in nfs_generic_pgio()
941 	nfs_pgio_rpcsetup(hdr, mirror->pg_count, desc->pg_ioflags, &cinfo);  in nfs_generic_pgio()
942 	desc->pg_rpc_callops = &nfs_pgio_common_ops;  in nfs_generic_pgio()
953 	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);  in nfs_generic_pg_pgios()
955 		desc->pg_error = -ENOMEM;  in nfs_generic_pg_pgios()
956 		return desc->pg_error;  in nfs_generic_pg_pgios()
961 		if (NFS_SERVER(hdr->inode)->nfs_client->cl_minorversion)  in nfs_generic_pg_pgios()
963 		ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),  in nfs_generic_pg_pgios()
965 					hdr->cred,  in nfs_generic_pg_pgios()
966 					NFS_PROTO(hdr->inode),  in nfs_generic_pg_pgios()
967 					desc->pg_rpc_callops,  in nfs_generic_pg_pgios()
968 					desc->pg_ioflags,  in nfs_generic_pg_pgios()
981 	kfree(desc->pg_mirrors_dynamic);  in nfs_pageio_alloc_mirrors()
982 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_alloc_mirrors()
984 		return desc->pg_mirrors_static;  in nfs_pageio_alloc_mirrors()
988 			nfs_pageio_mirror_init(&ret[i], desc->pg_bsize);  in nfs_pageio_alloc_mirrors()
989 		desc->pg_mirrors_dynamic = ret;  in nfs_pageio_alloc_mirrors()
995  * nfs_pageio_setup_mirroring - determine if mirroring is to be used
1003 	if (pgio->pg_ops->pg_get_mirror_count)  in nfs_pageio_setup_mirroring()
1004 		mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);  in nfs_pageio_setup_mirroring()
1005 	if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0)  in nfs_pageio_setup_mirroring()
1009 		pgio->pg_error = -EINVAL;  in nfs_pageio_setup_mirroring()
1013 	pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count);  in nfs_pageio_setup_mirroring()
1014 	if (pgio->pg_mirrors == NULL) {  in nfs_pageio_setup_mirroring()
1015 		pgio->pg_error = -ENOMEM;  in nfs_pageio_setup_mirroring()
1016 		pgio->pg_mirrors = pgio->pg_mirrors_static;  in nfs_pageio_setup_mirroring()
1019 	pgio->pg_mirror_count = mirror_count;  in nfs_pageio_setup_mirroring()
1024 	pgio->pg_mirror_count = 1;  in nfs_pageio_cleanup_mirroring()
1025 	pgio->pg_mirror_idx = 0;  in nfs_pageio_cleanup_mirroring()
1026 	pgio->pg_mirrors = pgio->pg_mirrors_static;  in nfs_pageio_cleanup_mirroring()
1027 	kfree(pgio->pg_mirrors_dynamic);  in nfs_pageio_cleanup_mirroring()
1028 	pgio->pg_mirrors_dynamic = NULL;  in nfs_pageio_cleanup_mirroring()
1034 	return l1->lockowner == l2->lockowner;  in nfs_match_lock_context()
1038  * nfs_coalesce_size - test two requests for compatibility
1044  * page data area they describe is contiguous, and that their RPC
1058 		flctx = d_inode(nfs_req_openctx(req)->dentry)->i_flctx;  in nfs_coalesce_size()
1060 		    !(list_empty_careful(&flctx->flc_posix) &&  in nfs_coalesce_size()
1061 		      list_empty_careful(&flctx->flc_flock)) &&  in nfs_coalesce_size()
1062 		    !nfs_match_lock_context(req->wb_lock_context,  in nfs_coalesce_size()
1063 					    prev->wb_lock_context))  in nfs_coalesce_size()
1065 		if (req_offset(req) != req_offset(prev) + prev->wb_bytes)  in nfs_coalesce_size()
1067 		if (req->wb_page == prev->wb_page) {  in nfs_coalesce_size()
1068 			if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes)  in nfs_coalesce_size()
1071 			if (req->wb_pgbase != 0 ||  in nfs_coalesce_size()
1072 			    prev->wb_pgbase + prev->wb_bytes != PAGE_SIZE)  in nfs_coalesce_size()
1076 	return pgio->pg_ops->pg_test(pgio, prev, req);  in nfs_coalesce_size()
1080  * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
1091 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_do_add_request()  local
1095 	if (list_empty(&mirror->pg_list)) {  in nfs_pageio_do_add_request()
1096 		if (desc->pg_ops->pg_init)  in nfs_pageio_do_add_request()
1097 			desc->pg_ops->pg_init(desc, req);  in nfs_pageio_do_add_request()
1098 		if (desc->pg_error < 0)  in nfs_pageio_do_add_request()
1100 		mirror->pg_base = req->wb_pgbase;  in nfs_pageio_do_add_request()
1101 		mirror->pg_count = 0;  in nfs_pageio_do_add_request()
1102 		mirror->pg_recoalesce = 0;  in nfs_pageio_do_add_request()
1104 		prev = nfs_list_entry(mirror->pg_list.prev);  in nfs_pageio_do_add_request()
1106 	if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {  in nfs_pageio_do_add_request()
1107 		if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)  in nfs_pageio_do_add_request()
1108 			desc->pg_error = -ETIMEDOUT;  in nfs_pageio_do_add_request()
1110 			desc->pg_error = -EIO;  in nfs_pageio_do_add_request()
1115 	if (size < req->wb_bytes)  in nfs_pageio_do_add_request()
1117 	nfs_list_move_request(req, &mirror->pg_list);  in nfs_pageio_do_add_request()
1118 	mirror->pg_count += req->wb_bytes;  in nfs_pageio_do_add_request()
1119 	return req->wb_bytes;  in nfs_pageio_do_add_request()
1127 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_doio()  local
1129 	if (!list_empty(&mirror->pg_list)) {  in nfs_pageio_doio()
1130 		int error = desc->pg_ops->pg_doio(desc);  in nfs_pageio_doio()
1132 			desc->pg_error = error;  in nfs_pageio_doio()
1133 		if (list_empty(&mirror->pg_list))  in nfs_pageio_doio()
1134 			mirror->pg_bytes_written += mirror->pg_count;  in nfs_pageio_doio()
1145 	desc->pg_completion_ops->error_cleanup(&head, desc->pg_error);  in nfs_pageio_cleanup_request()
1149  * __nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1163 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in __nfs_pageio_add_request()  local
1170 	subreq_size = subreq->wb_bytes;  in __nfs_pageio_add_request()
1177 			req->wb_pgbase += size;  in __nfs_pageio_add_request()
1178 			req->wb_bytes -= size;  in __nfs_pageio_add_request()
1179 			req->wb_offset += size;  in __nfs_pageio_add_request()
1180 			subreq_size = req->wb_bytes;  in __nfs_pageio_add_request()
1188 			subreq_size = req->wb_bytes;  in __nfs_pageio_add_request()
1194 			desc->pg_moreio = 1;  in __nfs_pageio_add_request()
1196 			if (desc->pg_error < 0 || mirror->pg_recoalesce)  in __nfs_pageio_add_request()
1202 		subreq = nfs_create_subreq(req, req->wb_pgbase,  in __nfs_pageio_add_request()
1203 				req->wb_offset, size);  in __nfs_pageio_add_request()
1212 	desc->pg_error = PTR_ERR(subreq);  in __nfs_pageio_add_request()
1219 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_do_recoalesce()  local
1223 		list_splice_init(&mirror->pg_list, &head);  in nfs_do_recoalesce()
1224 		mirror->pg_recoalesce = 0;  in nfs_do_recoalesce()
1232 			if (desc->pg_error < 0) {  in nfs_do_recoalesce()
1233 				list_splice_tail(&head, &mirror->pg_list);  in nfs_do_recoalesce()
1234 				mirror->pg_recoalesce = 1;  in nfs_do_recoalesce()
1239 	} while (mirror->pg_recoalesce);  in nfs_do_recoalesce()
1252 		if (desc->pg_error < 0)  in nfs_pageio_add_request_mirror()
1263 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_error_cleanup()  local
1265 	if (!desc->pg_error)  in nfs_pageio_error_cleanup()
1268 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_error_cleanup()
1269 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_error_cleanup()
1270 		desc->pg_completion_ops->error_cleanup(&mirror->pg_list,  in nfs_pageio_error_cleanup()
1271 				desc->pg_error);  in nfs_pageio_error_cleanup()
1282 	pgbase = req->wb_pgbase;  in nfs_pageio_add_request()
1283 	offset = req->wb_offset;  in nfs_pageio_add_request()
1284 	bytes = req->wb_bytes;  in nfs_pageio_add_request()
1287 	if (desc->pg_error < 0)  in nfs_pageio_add_request()
1290 	/* Create the mirror instances first, and fire them off */  in nfs_pageio_add_request()
1291 	for (midx = 1; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_add_request()
1299 			desc->pg_error = PTR_ERR(dupreq);  in nfs_pageio_add_request()
1322  * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1325  * @mirror_idx: pointer to mirror index
1330 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_complete_mirror()  local
1334 	mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_complete_mirror()
1338 		if (desc->pg_error < 0 || !mirror->pg_recoalesce)  in nfs_pageio_complete_mirror()
1347  * nfs_pageio_resend - Transfer requests to new descriptor and resend
1348  * @hdr - the pgio header to move request from
1349  * @desc - the pageio descriptor to add requests to
1361 	desc->pg_io_completion = hdr->io_completion;  in nfs_pageio_resend()
1362 	desc->pg_dreq = hdr->dreq;  in nfs_pageio_resend()
1363 	list_splice_init(&hdr->pages, &pages);  in nfs_pageio_resend()
1372 		int err = desc->pg_error < 0 ? desc->pg_error : -EIO;  in nfs_pageio_resend()
1373 		hdr->completion_ops->error_cleanup(&pages, err);  in nfs_pageio_resend()
1374 		nfs_set_pgio_error(hdr, err, hdr->io_start);  in nfs_pageio_resend()
1382  * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1389 	for (midx = 0; midx < desc->pg_mirror_count; midx++)  in nfs_pageio_complete()
1392 	if (desc->pg_error < 0)  in nfs_pageio_complete()
1394 	if (desc->pg_ops->pg_cleanup)  in nfs_pageio_complete()
1395 		desc->pg_ops->pg_cleanup(desc);  in nfs_pageio_complete()
1400  * nfs_pageio_cond_complete - Conditional I/O completion
1405  * on non-contiguous ranges of pages as that might deadlock. This
1412 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_cond_complete()  local
1416 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_cond_complete()
1417 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_cond_complete()
1418 		if (!list_empty(&mirror->pg_list)) {  in nfs_pageio_cond_complete()
1419 			prev = nfs_list_entry(mirror->pg_list.prev);  in nfs_pageio_cond_complete()
1420 			if (index != prev->wb_index + 1) {  in nfs_pageio_cond_complete()
1429  * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
1443 		return -ENOMEM;  in nfs_init_nfspagecache()