71 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
74 static inline void get_dreq(struct nfs_direct_req *dreq)  in get_dreq()  argument
76 	atomic_inc(&dreq->io_count);  in get_dreq()
79 static inline int put_dreq(struct nfs_direct_req *dreq)  in put_dreq()  argument
81 	return atomic_dec_and_test(&dreq->io_count);  in put_dreq()
85 nfs_direct_handle_truncated(struct nfs_direct_req *dreq,  in nfs_direct_handle_truncated()  argument
92 	if (dreq->max_count >= dreq_len) {  in nfs_direct_handle_truncated()
93 		dreq->max_count = dreq_len;  in nfs_direct_handle_truncated()
94 		if (dreq->count > dreq_len)  in nfs_direct_handle_truncated()
95 			dreq->count = dreq_len;  in nfs_direct_handle_truncated()
98 	if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && !dreq->error)  in nfs_direct_handle_truncated()
99 		dreq->error = hdr->error;  in nfs_direct_handle_truncated()
103 nfs_direct_count_bytes(struct nfs_direct_req *dreq,  in nfs_direct_count_bytes()  argument
109 	if (hdr_end > dreq->io_start)  in nfs_direct_count_bytes()
110 		dreq_len = hdr_end - dreq->io_start;  in nfs_direct_count_bytes()
112 	nfs_direct_handle_truncated(dreq, hdr, dreq_len);  in nfs_direct_count_bytes()
114 	if (dreq_len > dreq->max_count)  in nfs_direct_count_bytes()
115 		dreq_len = dreq->max_count;  in nfs_direct_count_bytes()
117 	if (dreq->count < dreq_len)  in nfs_direct_count_bytes()
118 		dreq->count = dreq_len;  in nfs_direct_count_bytes()
121 static void nfs_direct_truncate_request(struct nfs_direct_req *dreq,  in nfs_direct_truncate_request()  argument
125 	size_t req_start = (size_t)(offs - dreq->io_start);  in nfs_direct_truncate_request()
127 	if (req_start < dreq->max_count)  in nfs_direct_truncate_request()
128 		dreq->max_count = req_start;  in nfs_direct_truncate_request()
129 	if (req_start < dreq->count)  in nfs_direct_truncate_request()
130 		dreq->count = req_start;  in nfs_direct_truncate_request()
163 			      struct nfs_direct_req *dreq)  in nfs_init_cinfo_from_dreq()  argument
165 	cinfo->inode = dreq->inode;  in nfs_init_cinfo_from_dreq()
166 	cinfo->mds = &dreq->mds_cinfo;  in nfs_init_cinfo_from_dreq()
167 	cinfo->ds = &dreq->ds_cinfo;  in nfs_init_cinfo_from_dreq()
168 	cinfo->dreq = dreq;  in nfs_init_cinfo_from_dreq()
174 	struct nfs_direct_req *dreq;  in nfs_direct_req_alloc()  local
176 	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);  in nfs_direct_req_alloc()
177 	if (!dreq)  in nfs_direct_req_alloc()
180 	kref_init(&dreq->kref);  in nfs_direct_req_alloc()
181 	kref_get(&dreq->kref);  in nfs_direct_req_alloc()
182 	init_completion(&dreq->completion);  in nfs_direct_req_alloc()
183 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);  in nfs_direct_req_alloc()
184 	pnfs_init_ds_commit_info(&dreq->ds_cinfo);  in nfs_direct_req_alloc()
185 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);  in nfs_direct_req_alloc()
186 	spin_lock_init(&dreq->lock);  in nfs_direct_req_alloc()
188 	return dreq;  in nfs_direct_req_alloc()
193 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);  in nfs_direct_req_free()  local
195 	pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);  in nfs_direct_req_free()
196 	if (dreq->l_ctx != NULL)  in nfs_direct_req_free()
197 		nfs_put_lock_context(dreq->l_ctx);  in nfs_direct_req_free()
198 	if (dreq->ctx != NULL)  in nfs_direct_req_free()
199 		put_nfs_open_context(dreq->ctx);  in nfs_direct_req_free()
200 	kmem_cache_free(nfs_direct_cachep, dreq);  in nfs_direct_req_free()
203 static void nfs_direct_req_release(struct nfs_direct_req *dreq)  in nfs_direct_req_release()  argument
205 	kref_put(&dreq->kref, nfs_direct_req_free);  in nfs_direct_req_release()
208 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)  in nfs_dreq_bytes_left()  argument
210 	return dreq->bytes_left;  in nfs_dreq_bytes_left()
217 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)  in nfs_direct_wait()  argument
222 	if (dreq->iocb)  in nfs_direct_wait()
225 	result = wait_for_completion_killable(&dreq->completion);  in nfs_direct_wait()
228 		result = dreq->count;  in nfs_direct_wait()
229 		WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_wait()
232 		result = dreq->error;  in nfs_direct_wait()
242 static void nfs_direct_complete(struct nfs_direct_req *dreq)  in nfs_direct_complete()  argument
244 	struct inode *inode = dreq->inode;  in nfs_direct_complete()
248 	if (dreq->iocb) {  in nfs_direct_complete()
249 		long res = (long) dreq->error;  in nfs_direct_complete()
250 		if (dreq->count != 0) {  in nfs_direct_complete()
251 			res = (long) dreq->count;  in nfs_direct_complete()
252 			WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_complete()
254 		dreq->iocb->ki_complete(dreq->iocb, res);  in nfs_direct_complete()
257 	complete(&dreq->completion);  in nfs_direct_complete()
259 	nfs_direct_req_release(dreq);  in nfs_direct_complete()
265 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_read_completion()  local
267 	spin_lock(&dreq->lock);  in nfs_direct_read_completion()
269 		spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
273 	nfs_direct_count_bytes(dreq, hdr);  in nfs_direct_read_completion()
274 	spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
281 		    (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))  in nfs_direct_read_completion()
288 	if (put_dreq(dreq))  in nfs_direct_read_completion()
289 		nfs_direct_complete(dreq);  in nfs_direct_read_completion()
306 	get_dreq(hdr->dreq);  in nfs_direct_pgio_init()
323 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_read_schedule_iovec()  argument
328 	struct inode *inode = dreq->inode;  in nfs_direct_read_schedule_iovec()
333 	nfs_pageio_init_read(&desc, dreq->inode, false,  in nfs_direct_read_schedule_iovec()
335 	get_dreq(dreq);  in nfs_direct_read_schedule_iovec()
336 	desc.pg_dreq = dreq;  in nfs_direct_read_schedule_iovec()
356 			req = nfs_page_create_from_page(dreq->ctx, pagevec[i],  in nfs_direct_read_schedule_iovec()
371 			dreq->bytes_left -= req_len;  in nfs_direct_read_schedule_iovec()
387 		nfs_direct_req_release(dreq);  in nfs_direct_read_schedule_iovec()
391 	if (put_dreq(dreq))  in nfs_direct_read_schedule_iovec()
392 		nfs_direct_complete(dreq);  in nfs_direct_read_schedule_iovec()
422 	struct nfs_direct_req *dreq;  in nfs_file_direct_read()  local
438 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_read()
439 	if (dreq == NULL)  in nfs_file_direct_read()
442 	dreq->inode = inode;  in nfs_file_direct_read()
443 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_read()
444 	dreq->io_start = iocb->ki_pos;  in nfs_file_direct_read()
445 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_read()
446 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_read()
449 		nfs_direct_req_release(dreq);  in nfs_file_direct_read()
452 	dreq->l_ctx = l_ctx;  in nfs_file_direct_read()
454 		dreq->iocb = iocb;  in nfs_file_direct_read()
457 		dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;  in nfs_file_direct_read()
463 	requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);  in nfs_file_direct_read()
469 		result = nfs_direct_wait(dreq);  in nfs_file_direct_read()
480 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
540 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)  in nfs_direct_write_reschedule()  argument
547 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule()
548 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_reschedule()
550 	nfs_direct_join_group(&reqs, &cinfo, dreq->inode);  in nfs_direct_write_reschedule()
552 	nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);  in nfs_direct_write_reschedule()
553 	get_dreq(dreq);  in nfs_direct_write_reschedule()
555 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,  in nfs_direct_write_reschedule()
557 	desc.pg_dreq = dreq;  in nfs_direct_write_reschedule()
564 			spin_lock(&dreq->lock);  in nfs_direct_write_reschedule()
565 			if (dreq->error < 0) {  in nfs_direct_write_reschedule()
566 				desc.pg_error = dreq->error;  in nfs_direct_write_reschedule()
568 				dreq->flags = 0;  in nfs_direct_write_reschedule()
571 				dreq->error = desc.pg_error;  in nfs_direct_write_reschedule()
573 				dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_write_reschedule()
574 			spin_unlock(&dreq->lock);  in nfs_direct_write_reschedule()
588 			spin_lock(&dreq->lock);  in nfs_direct_write_reschedule()
589 			nfs_direct_truncate_request(dreq, req);  in nfs_direct_write_reschedule()
590 			spin_unlock(&dreq->lock);  in nfs_direct_write_reschedule()
595 	if (put_dreq(dreq))  in nfs_direct_write_reschedule()
596 		nfs_direct_write_complete(dreq);  in nfs_direct_write_reschedule()
602 	struct nfs_direct_req *dreq = data->dreq;  in nfs_direct_commit_complete()  local
607 	trace_nfs_direct_commit_complete(dreq);  in nfs_direct_commit_complete()
611 		dreq->error = status;  in nfs_direct_commit_complete()
612 		dreq->flags = NFS_ODIRECT_DONE;  in nfs_direct_commit_complete()
614 		status = dreq->error;  in nfs_direct_commit_complete()
617 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_complete()
623 			spin_lock(&dreq->lock);  in nfs_direct_commit_complete()
624 			nfs_direct_truncate_request(dreq, req);  in nfs_direct_commit_complete()
625 			spin_unlock(&dreq->lock);  in nfs_direct_commit_complete()
628 			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
641 		nfs_direct_write_complete(dreq);  in nfs_direct_commit_complete()
647 	struct nfs_direct_req *dreq = cinfo->dreq;  in nfs_direct_resched_write()  local
649 	trace_nfs_direct_resched_write(dreq);  in nfs_direct_resched_write()
651 	spin_lock(&dreq->lock);  in nfs_direct_resched_write()
652 	if (dreq->flags != NFS_ODIRECT_DONE)  in nfs_direct_resched_write()
653 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_resched_write()
654 	spin_unlock(&dreq->lock);  in nfs_direct_resched_write()
663 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)  in nfs_direct_commit_schedule()  argument
669 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_schedule()
670 	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);  in nfs_direct_commit_schedule()
671 	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);  in nfs_direct_commit_schedule()
673 		nfs_direct_write_reschedule(dreq);  in nfs_direct_commit_schedule()
676 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)  in nfs_direct_write_clear_reqs()  argument
682 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_clear_reqs()
683 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_clear_reqs()
688 		nfs_direct_truncate_request(dreq, req);  in nfs_direct_write_clear_reqs()
696 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);  in nfs_direct_write_schedule_work()  local
697 	int flags = dreq->flags;  in nfs_direct_write_schedule_work()
699 	dreq->flags = 0;  in nfs_direct_write_schedule_work()
702 			nfs_direct_commit_schedule(dreq);  in nfs_direct_write_schedule_work()
705 			nfs_direct_write_reschedule(dreq);  in nfs_direct_write_schedule_work()
708 			nfs_direct_write_clear_reqs(dreq);  in nfs_direct_write_schedule_work()
709 			nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);  in nfs_direct_write_schedule_work()
710 			nfs_direct_complete(dreq);  in nfs_direct_write_schedule_work()
714 static void nfs_direct_write_complete(struct nfs_direct_req *dreq)  in nfs_direct_write_complete()  argument
716 	trace_nfs_direct_write_complete(dreq);  in nfs_direct_write_complete()
717 	queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */  in nfs_direct_write_complete()
722 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_completion()  local
727 	trace_nfs_direct_write_completion(dreq);  in nfs_direct_write_completion()
729 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_completion()
731 	spin_lock(&dreq->lock);  in nfs_direct_write_completion()
733 		spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
737 	nfs_direct_count_bytes(dreq, hdr);  in nfs_direct_write_completion()
740 		if (!dreq->flags)  in nfs_direct_write_completion()
741 			dreq->flags = NFS_ODIRECT_DO_COMMIT;  in nfs_direct_write_completion()
742 		flags = dreq->flags;  in nfs_direct_write_completion()
744 	spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
764 	if (put_dreq(dreq))  in nfs_direct_write_completion()
765 		nfs_direct_write_complete(dreq);  in nfs_direct_write_completion()
782 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_reschedule_io()  local
786 	trace_nfs_direct_write_reschedule_io(dreq);  in nfs_direct_write_reschedule_io()
788 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule_io()
789 	spin_lock(&dreq->lock);  in nfs_direct_write_reschedule_io()
790 	if (dreq->error == 0)  in nfs_direct_write_reschedule_io()
791 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_write_reschedule_io()
793 	spin_unlock(&dreq->lock);  in nfs_direct_write_reschedule_io()
821 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_write_schedule_iovec()  argument
826 	struct inode *inode = dreq->inode;  in nfs_direct_write_schedule_iovec()
833 	trace_nfs_direct_write_schedule_iovec(dreq);  in nfs_direct_write_schedule_iovec()
837 	desc.pg_dreq = dreq;  in nfs_direct_write_schedule_iovec()
838 	get_dreq(dreq);  in nfs_direct_write_schedule_iovec()
859 			req = nfs_page_create_from_page(dreq->ctx, pagevec[i],  in nfs_direct_write_schedule_iovec()
876 			dreq->bytes_left -= req_len;  in nfs_direct_write_schedule_iovec()
895 			nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_schedule_iovec()
896 			spin_lock(&dreq->lock);  in nfs_direct_write_schedule_iovec()
897 			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_write_schedule_iovec()
898 			spin_unlock(&dreq->lock);  in nfs_direct_write_schedule_iovec()
917 		nfs_direct_req_release(dreq);  in nfs_direct_write_schedule_iovec()
921 	if (put_dreq(dreq))  in nfs_direct_write_schedule_iovec()
922 		nfs_direct_write_complete(dreq);  in nfs_direct_write_schedule_iovec()
955 	struct nfs_direct_req *dreq;  in nfs_file_direct_write()  local
978 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_write()
979 	if (!dreq)  in nfs_file_direct_write()
982 	dreq->inode = inode;  in nfs_file_direct_write()
983 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_write()
984 	dreq->io_start = pos;  in nfs_file_direct_write()
985 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_write()
986 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_write()
989 		nfs_direct_req_release(dreq);  in nfs_file_direct_write()
992 	dreq->l_ctx = l_ctx;  in nfs_file_direct_write()
994 		dreq->iocb = iocb;  in nfs_file_direct_write()
995 	pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);  in nfs_file_direct_write()
998 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,  in nfs_file_direct_write()
1003 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,  in nfs_file_direct_write()
1015 		result = nfs_direct_wait(dreq);  in nfs_file_direct_write()
1028 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()