71 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
74 static inline void get_dreq(struct nfs_direct_req *dreq)  in get_dreq()  argument
76 	atomic_inc(&dreq->io_count);  in get_dreq()
79 static inline int put_dreq(struct nfs_direct_req *dreq)  in put_dreq()  argument
81 	return atomic_dec_and_test(&dreq->io_count);  in put_dreq()
85 nfs_direct_handle_truncated(struct nfs_direct_req *dreq,  in nfs_direct_handle_truncated()  argument
92 	if (dreq->max_count >= dreq_len) {  in nfs_direct_handle_truncated()
93 		dreq->max_count = dreq_len;  in nfs_direct_handle_truncated()
94 		if (dreq->count > dreq_len)  in nfs_direct_handle_truncated()
95 			dreq->count = dreq_len;  in nfs_direct_handle_truncated()
98 			dreq->error = hdr->error;  in nfs_direct_handle_truncated()
100 			dreq->error = 0;  in nfs_direct_handle_truncated()
105 nfs_direct_count_bytes(struct nfs_direct_req *dreq,  in nfs_direct_count_bytes()  argument
111 	if (hdr_end > dreq->io_start)  in nfs_direct_count_bytes()
112 		dreq_len = hdr_end - dreq->io_start;  in nfs_direct_count_bytes()
114 	nfs_direct_handle_truncated(dreq, hdr, dreq_len);  in nfs_direct_count_bytes()
116 	if (dreq_len > dreq->max_count)  in nfs_direct_count_bytes()
117 		dreq_len = dreq->max_count;  in nfs_direct_count_bytes()
119 	if (dreq->count < dreq_len)  in nfs_direct_count_bytes()
120 		dreq->count = dreq_len;  in nfs_direct_count_bytes()
153 			      struct nfs_direct_req *dreq)  in nfs_init_cinfo_from_dreq()  argument
155 	cinfo->inode = dreq->inode;  in nfs_init_cinfo_from_dreq()
156 	cinfo->mds = &dreq->mds_cinfo;  in nfs_init_cinfo_from_dreq()
157 	cinfo->ds = &dreq->ds_cinfo;  in nfs_init_cinfo_from_dreq()
158 	cinfo->dreq = dreq;  in nfs_init_cinfo_from_dreq()
164 	struct nfs_direct_req *dreq;  in nfs_direct_req_alloc()  local
166 	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);  in nfs_direct_req_alloc()
167 	if (!dreq)  in nfs_direct_req_alloc()
170 	kref_init(&dreq->kref);  in nfs_direct_req_alloc()
171 	kref_get(&dreq->kref);  in nfs_direct_req_alloc()
172 	init_completion(&dreq->completion);  in nfs_direct_req_alloc()
173 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);  in nfs_direct_req_alloc()
174 	pnfs_init_ds_commit_info(&dreq->ds_cinfo);  in nfs_direct_req_alloc()
175 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);  in nfs_direct_req_alloc()
176 	spin_lock_init(&dreq->lock);  in nfs_direct_req_alloc()
178 	return dreq;  in nfs_direct_req_alloc()
183 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);  in nfs_direct_req_free()  local
185 	pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);  in nfs_direct_req_free()
186 	if (dreq->l_ctx != NULL)  in nfs_direct_req_free()
187 		nfs_put_lock_context(dreq->l_ctx);  in nfs_direct_req_free()
188 	if (dreq->ctx != NULL)  in nfs_direct_req_free()
189 		put_nfs_open_context(dreq->ctx);  in nfs_direct_req_free()
190 	kmem_cache_free(nfs_direct_cachep, dreq);  in nfs_direct_req_free()
193 static void nfs_direct_req_release(struct nfs_direct_req *dreq)  in nfs_direct_req_release()  argument
195 	kref_put(&dreq->kref, nfs_direct_req_free);  in nfs_direct_req_release()
198 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)  in nfs_dreq_bytes_left()  argument
200 	return dreq->bytes_left;  in nfs_dreq_bytes_left()
207 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)  in nfs_direct_wait()  argument
212 	if (dreq->iocb)  in nfs_direct_wait()
215 	result = wait_for_completion_killable(&dreq->completion);  in nfs_direct_wait()
218 		result = dreq->count;  in nfs_direct_wait()
219 		WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_wait()
222 		result = dreq->error;  in nfs_direct_wait()
232 static void nfs_direct_complete(struct nfs_direct_req *dreq)  in nfs_direct_complete()  argument
234 	struct inode *inode = dreq->inode;  in nfs_direct_complete()
238 	if (dreq->iocb) {  in nfs_direct_complete()
239 		long res = (long) dreq->error;  in nfs_direct_complete()
240 		if (dreq->count != 0) {  in nfs_direct_complete()
241 			res = (long) dreq->count;  in nfs_direct_complete()
242 			WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_complete()
244 		dreq->iocb->ki_complete(dreq->iocb, res);  in nfs_direct_complete()
247 	complete(&dreq->completion);  in nfs_direct_complete()
249 	nfs_direct_req_release(dreq);  in nfs_direct_complete()
255 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_read_completion()  local
257 	spin_lock(&dreq->lock);  in nfs_direct_read_completion()
259 		spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
263 	nfs_direct_count_bytes(dreq, hdr);  in nfs_direct_read_completion()
264 	spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
271 		    (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))  in nfs_direct_read_completion()
278 	if (put_dreq(dreq))  in nfs_direct_read_completion()
279 		nfs_direct_complete(dreq);  in nfs_direct_read_completion()
296 	get_dreq(hdr->dreq);  in nfs_direct_pgio_init()
313 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_read_schedule_iovec()  argument
318 	struct inode *inode = dreq->inode;  in nfs_direct_read_schedule_iovec()
323 	nfs_pageio_init_read(&desc, dreq->inode, false,  in nfs_direct_read_schedule_iovec()
325 	get_dreq(dreq);  in nfs_direct_read_schedule_iovec()
326 	desc.pg_dreq = dreq;  in nfs_direct_read_schedule_iovec()
346 			req = nfs_create_request(dreq->ctx, pagevec[i],  in nfs_direct_read_schedule_iovec()
363 			dreq->bytes_left -= req_len;  in nfs_direct_read_schedule_iovec()
379 		nfs_direct_req_release(dreq);  in nfs_direct_read_schedule_iovec()
383 	if (put_dreq(dreq))  in nfs_direct_read_schedule_iovec()
384 		nfs_direct_complete(dreq);  in nfs_direct_read_schedule_iovec()
414 	struct nfs_direct_req *dreq;  in nfs_file_direct_read()  local
430 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_read()
431 	if (dreq == NULL)  in nfs_file_direct_read()
434 	dreq->inode = inode;  in nfs_file_direct_read()
435 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_read()
436 	dreq->io_start = iocb->ki_pos;  in nfs_file_direct_read()
437 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_read()
438 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_read()
441 		nfs_direct_req_release(dreq);  in nfs_file_direct_read()
444 	dreq->l_ctx = l_ctx;  in nfs_file_direct_read()
446 		dreq->iocb = iocb;  in nfs_file_direct_read()
449 		dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;  in nfs_file_direct_read()
455 	requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);  in nfs_file_direct_read()
461 		result = nfs_direct_wait(dreq);  in nfs_file_direct_read()
472 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
506 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)  in nfs_direct_write_reschedule()  argument
514 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule()
515 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_reschedule()
517 	nfs_direct_join_group(&reqs, dreq->inode);  in nfs_direct_write_reschedule()
519 	dreq->count = 0;  in nfs_direct_write_reschedule()
520 	dreq->max_count = 0;  in nfs_direct_write_reschedule()
522 		dreq->max_count += req->wb_bytes;  in nfs_direct_write_reschedule()
523 	nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);  in nfs_direct_write_reschedule()
524 	get_dreq(dreq);  in nfs_direct_write_reschedule()
526 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,  in nfs_direct_write_reschedule()
528 	desc.pg_dreq = dreq;  in nfs_direct_write_reschedule()
536 			dreq->flags = 0;  in nfs_direct_write_reschedule()
538 				dreq->error = desc.pg_error;  in nfs_direct_write_reschedule()
540 				dreq->error = -EIO;  in nfs_direct_write_reschedule()
553 	if (put_dreq(dreq))  in nfs_direct_write_reschedule()
554 		nfs_direct_write_complete(dreq);  in nfs_direct_write_reschedule()
560 	struct nfs_direct_req *dreq = data->dreq;  in nfs_direct_commit_complete()  local
565 	trace_nfs_direct_commit_complete(dreq);  in nfs_direct_commit_complete()
569 		dreq->error = status;  in nfs_direct_commit_complete()
570 		dreq->max_count = 0;  in nfs_direct_commit_complete()
571 		dreq->count = 0;  in nfs_direct_commit_complete()
572 		dreq->flags = NFS_ODIRECT_DONE;  in nfs_direct_commit_complete()
574 		status = dreq->error;  in nfs_direct_commit_complete()
577 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_complete()
583 			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
596 		nfs_direct_write_complete(dreq);  in nfs_direct_commit_complete()
602 	struct nfs_direct_req *dreq = cinfo->dreq;  in nfs_direct_resched_write()  local
604 	trace_nfs_direct_resched_write(dreq);  in nfs_direct_resched_write()
606 	spin_lock(&dreq->lock);  in nfs_direct_resched_write()
607 	if (dreq->flags != NFS_ODIRECT_DONE)  in nfs_direct_resched_write()
608 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_resched_write()
609 	spin_unlock(&dreq->lock);  in nfs_direct_resched_write()
618 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)  in nfs_direct_commit_schedule()  argument
624 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_schedule()
625 	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);  in nfs_direct_commit_schedule()
626 	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);  in nfs_direct_commit_schedule()
628 		nfs_direct_write_reschedule(dreq);  in nfs_direct_commit_schedule()
631 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)  in nfs_direct_write_clear_reqs()  argument
637 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_clear_reqs()
638 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_clear_reqs()
650 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);  in nfs_direct_write_schedule_work()  local
651 	int flags = dreq->flags;  in nfs_direct_write_schedule_work()
653 	dreq->flags = 0;  in nfs_direct_write_schedule_work()
656 			nfs_direct_commit_schedule(dreq);  in nfs_direct_write_schedule_work()
659 			nfs_direct_write_reschedule(dreq);  in nfs_direct_write_schedule_work()
662 			nfs_direct_write_clear_reqs(dreq);  in nfs_direct_write_schedule_work()
663 			nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);  in nfs_direct_write_schedule_work()
664 			nfs_direct_complete(dreq);  in nfs_direct_write_schedule_work()
668 static void nfs_direct_write_complete(struct nfs_direct_req *dreq)  in nfs_direct_write_complete()  argument
670 	trace_nfs_direct_write_complete(dreq);  in nfs_direct_write_complete()
671 	queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */  in nfs_direct_write_complete()
676 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_completion()  local
681 	trace_nfs_direct_write_completion(dreq);  in nfs_direct_write_completion()
683 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_completion()
685 	spin_lock(&dreq->lock);  in nfs_direct_write_completion()
687 		spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
691 	nfs_direct_count_bytes(dreq, hdr);  in nfs_direct_write_completion()
693 		if (!dreq->flags)  in nfs_direct_write_completion()
694 			dreq->flags = NFS_ODIRECT_DO_COMMIT;  in nfs_direct_write_completion()
695 		flags = dreq->flags;  in nfs_direct_write_completion()
697 	spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
717 	if (put_dreq(dreq))  in nfs_direct_write_completion()
718 		nfs_direct_write_complete(dreq);  in nfs_direct_write_completion()
735 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_reschedule_io()  local
737 	trace_nfs_direct_write_reschedule_io(dreq);  in nfs_direct_write_reschedule_io()
739 	spin_lock(&dreq->lock);  in nfs_direct_write_reschedule_io()
740 	if (dreq->error == 0) {  in nfs_direct_write_reschedule_io()
741 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_write_reschedule_io()
747 	spin_unlock(&dreq->lock);  in nfs_direct_write_reschedule_io()
769 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_write_schedule_iovec()  argument
774 	struct inode *inode = dreq->inode;  in nfs_direct_write_schedule_iovec()
779 	trace_nfs_direct_write_schedule_iovec(dreq);  in nfs_direct_write_schedule_iovec()
783 	desc.pg_dreq = dreq;  in nfs_direct_write_schedule_iovec()
784 	get_dreq(dreq);  in nfs_direct_write_schedule_iovec()
805 			req = nfs_create_request(dreq->ctx, pagevec[i],  in nfs_direct_write_schedule_iovec()
830 			dreq->bytes_left -= req_len;  in nfs_direct_write_schedule_iovec()
845 		nfs_direct_req_release(dreq);  in nfs_direct_write_schedule_iovec()
849 	if (put_dreq(dreq))  in nfs_direct_write_schedule_iovec()
850 		nfs_direct_write_complete(dreq);  in nfs_direct_write_schedule_iovec()
883 	struct nfs_direct_req *dreq;  in nfs_file_direct_write()  local
906 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_write()
907 	if (!dreq)  in nfs_file_direct_write()
910 	dreq->inode = inode;  in nfs_file_direct_write()
911 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_write()
912 	dreq->io_start = pos;  in nfs_file_direct_write()
913 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_write()
914 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_write()
917 		nfs_direct_req_release(dreq);  in nfs_file_direct_write()
920 	dreq->l_ctx = l_ctx;  in nfs_file_direct_write()
922 		dreq->iocb = iocb;  in nfs_file_direct_write()
923 	pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);  in nfs_file_direct_write()
926 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,  in nfs_file_direct_write()
931 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,  in nfs_file_direct_write()
943 		result = nfs_direct_wait(dreq);  in nfs_file_direct_write()
956 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()