108 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
111 static inline void get_dreq(struct nfs_direct_req *dreq)  in get_dreq()  argument
113 	atomic_inc(&dreq->io_count);  in get_dreq()
116 static inline int put_dreq(struct nfs_direct_req *dreq)  in put_dreq()  argument
118 	return atomic_dec_and_test(&dreq->io_count);  in put_dreq()
122 nfs_direct_good_bytes(struct nfs_direct_req *dreq, struct nfs_pgio_header *hdr)  in nfs_direct_good_bytes()  argument
127 	WARN_ON_ONCE(dreq->count >= dreq->max_count);  in nfs_direct_good_bytes()
129 	if (dreq->mirror_count == 1) {  in nfs_direct_good_bytes()
130 		dreq->mirrors[hdr->pgio_mirror_idx].count += hdr->good_bytes;  in nfs_direct_good_bytes()
131 		dreq->count += hdr->good_bytes;  in nfs_direct_good_bytes()
134 		count = dreq->mirrors[hdr->pgio_mirror_idx].count;  in nfs_direct_good_bytes()
135 		if (count + dreq->io_start < hdr->io_start + hdr->good_bytes) {  in nfs_direct_good_bytes()
136 			count = hdr->io_start + hdr->good_bytes - dreq->io_start;  in nfs_direct_good_bytes()
137 			dreq->mirrors[hdr->pgio_mirror_idx].count = count;  in nfs_direct_good_bytes()
141 		count = dreq->mirrors[0].count;  in nfs_direct_good_bytes()
143 		for (i = 1; i < dreq->mirror_count; i++)  in nfs_direct_good_bytes()
144 			count = min(count, dreq->mirrors[i].count);  in nfs_direct_good_bytes()
146 		dreq->count = count;  in nfs_direct_good_bytes()
159 nfs_direct_select_verf(struct nfs_direct_req *dreq,  in nfs_direct_select_verf()  argument
163 	struct nfs_writeverf *verfp = &dreq->verf;  in nfs_direct_select_verf()
170 	if (ds_clp && dreq->ds_cinfo.nbuckets > 0) {  in nfs_direct_select_verf()
171 		if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)  in nfs_direct_select_verf()
172 			verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;  in nfs_direct_select_verf()
188 static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,  in nfs_direct_set_hdr_verf()  argument
193 	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);  in nfs_direct_set_hdr_verf()
214 static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,  in nfs_direct_set_or_cmp_hdr_verf()  argument
219 	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);  in nfs_direct_set_or_cmp_hdr_verf()
221 		nfs_direct_set_hdr_verf(dreq, hdr);  in nfs_direct_set_or_cmp_hdr_verf()
235 static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,  in nfs_direct_cmp_commit_data_verf()  argument
240 	verfp = nfs_direct_select_verf(dreq, data->ds_clp,  in nfs_direct_cmp_commit_data_verf()
283 			      struct nfs_direct_req *dreq)  in nfs_init_cinfo_from_dreq()  argument
285 	cinfo->inode = dreq->inode;  in nfs_init_cinfo_from_dreq()
286 	cinfo->mds = &dreq->mds_cinfo;  in nfs_init_cinfo_from_dreq()
287 	cinfo->ds = &dreq->ds_cinfo;  in nfs_init_cinfo_from_dreq()
288 	cinfo->dreq = dreq;  in nfs_init_cinfo_from_dreq()
292 static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,  in nfs_direct_setup_mirroring()  argument
301 	dreq->mirror_count = mirror_count;  in nfs_direct_setup_mirroring()
306 	struct nfs_direct_req *dreq;  in nfs_direct_req_alloc()  local
308 	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);  in nfs_direct_req_alloc()
309 	if (!dreq)  in nfs_direct_req_alloc()
312 	kref_init(&dreq->kref);  in nfs_direct_req_alloc()
313 	kref_get(&dreq->kref);  in nfs_direct_req_alloc()
314 	init_completion(&dreq->completion);  in nfs_direct_req_alloc()
315 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);  in nfs_direct_req_alloc()
316 	dreq->verf.committed = NFS_INVALID_STABLE_HOW;	/* not set yet */  in nfs_direct_req_alloc()
317 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);  in nfs_direct_req_alloc()
318 	dreq->mirror_count = 1;  in nfs_direct_req_alloc()
319 	spin_lock_init(&dreq->lock);  in nfs_direct_req_alloc()
321 	return dreq;  in nfs_direct_req_alloc()
326 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);  in nfs_direct_req_free()  local
328 	nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);  in nfs_direct_req_free()
329 	if (dreq->l_ctx != NULL)  in nfs_direct_req_free()
330 		nfs_put_lock_context(dreq->l_ctx);  in nfs_direct_req_free()
331 	if (dreq->ctx != NULL)  in nfs_direct_req_free()
332 		put_nfs_open_context(dreq->ctx);  in nfs_direct_req_free()
333 	kmem_cache_free(nfs_direct_cachep, dreq);  in nfs_direct_req_free()
336 static void nfs_direct_req_release(struct nfs_direct_req *dreq)  in nfs_direct_req_release()  argument
338 	kref_put(&dreq->kref, nfs_direct_req_free);  in nfs_direct_req_release()
341 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)  in nfs_dreq_bytes_left()  argument
343 	return dreq->bytes_left;  in nfs_dreq_bytes_left()
350 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)  in nfs_direct_wait()  argument
355 	if (dreq->iocb)  in nfs_direct_wait()
358 	result = wait_for_completion_killable(&dreq->completion);  in nfs_direct_wait()
361 		result = dreq->count;  in nfs_direct_wait()
362 		WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_wait()
365 		result = dreq->error;  in nfs_direct_wait()
375 static void nfs_direct_complete(struct nfs_direct_req *dreq)  in nfs_direct_complete()  argument
377 	struct inode *inode = dreq->inode;  in nfs_direct_complete()
381 	if (dreq->iocb) {  in nfs_direct_complete()
382 		long res = (long) dreq->error;  in nfs_direct_complete()
383 		if (dreq->count != 0) {  in nfs_direct_complete()
384 			res = (long) dreq->count;  in nfs_direct_complete()
385 			WARN_ON_ONCE(dreq->count < 0);  in nfs_direct_complete()
387 		dreq->iocb->ki_complete(dreq->iocb, res, 0);  in nfs_direct_complete()
390 	complete(&dreq->completion);  in nfs_direct_complete()
392 	nfs_direct_req_release(dreq);  in nfs_direct_complete()
398 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_read_completion()  local
403 	spin_lock(&dreq->lock);  in nfs_direct_read_completion()
405 		dreq->error = hdr->error;  in nfs_direct_read_completion()
407 		nfs_direct_good_bytes(dreq, hdr);  in nfs_direct_read_completion()
409 	spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
422 	if (put_dreq(dreq))  in nfs_direct_read_completion()
423 		nfs_direct_complete(dreq);  in nfs_direct_read_completion()
440 	get_dreq(hdr->dreq);  in nfs_direct_pgio_init()
457 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_read_schedule_iovec()  argument
462 	struct inode *inode = dreq->inode;  in nfs_direct_read_schedule_iovec()
467 	nfs_pageio_init_read(&desc, dreq->inode, false,  in nfs_direct_read_schedule_iovec()
469 	get_dreq(dreq);  in nfs_direct_read_schedule_iovec()
470 	desc.pg_dreq = dreq;  in nfs_direct_read_schedule_iovec()
491 			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,  in nfs_direct_read_schedule_iovec()
508 			dreq->bytes_left -= req_len;  in nfs_direct_read_schedule_iovec()
524 		nfs_direct_req_release(dreq);  in nfs_direct_read_schedule_iovec()
528 	if (put_dreq(dreq))  in nfs_direct_read_schedule_iovec()
529 		nfs_direct_complete(dreq);  in nfs_direct_read_schedule_iovec()
557 	struct nfs_direct_req *dreq;  in nfs_file_direct_read()  local
573 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_read()
574 	if (dreq == NULL)  in nfs_file_direct_read()
577 	dreq->inode = inode;  in nfs_file_direct_read()
578 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_read()
579 	dreq->io_start = iocb->ki_pos;  in nfs_file_direct_read()
580 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_read()
581 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_read()
586 	dreq->l_ctx = l_ctx;  in nfs_file_direct_read()
588 		dreq->iocb = iocb;  in nfs_file_direct_read()
593 	requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);  in nfs_file_direct_read()
598 		result = nfs_direct_wait(dreq);  in nfs_file_direct_read()
609 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
628 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)  in nfs_direct_write_reschedule()  argument
637 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule()
638 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_reschedule()
640 	dreq->count = 0;  in nfs_direct_write_reschedule()
641 	dreq->verf.committed = NFS_INVALID_STABLE_HOW;  in nfs_direct_write_reschedule()
642 	nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);  in nfs_direct_write_reschedule()
643 	for (i = 0; i < dreq->mirror_count; i++)  in nfs_direct_write_reschedule()
644 		dreq->mirrors[i].count = 0;  in nfs_direct_write_reschedule()
645 	get_dreq(dreq);  in nfs_direct_write_reschedule()
647 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,  in nfs_direct_write_reschedule()
649 	desc.pg_dreq = dreq;  in nfs_direct_write_reschedule()
652 	nfs_direct_setup_mirroring(dreq, &desc, req);  in nfs_direct_write_reschedule()
663 			dreq->flags = 0;  in nfs_direct_write_reschedule()
665 				dreq->error = desc.pg_error;  in nfs_direct_write_reschedule()
667 				dreq->error = -EIO;  in nfs_direct_write_reschedule()
681 	if (put_dreq(dreq))  in nfs_direct_write_reschedule()
682 		nfs_direct_write_complete(dreq);  in nfs_direct_write_reschedule()
687 	struct nfs_direct_req *dreq = data->dreq;  in nfs_direct_commit_complete()  local
692 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_complete()
693 	if (status < 0 || nfs_direct_cmp_commit_data_verf(dreq, data))  in nfs_direct_commit_complete()
694 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
699 		if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {  in nfs_direct_commit_complete()
708 		nfs_direct_write_complete(dreq);  in nfs_direct_commit_complete()
714 	struct nfs_direct_req *dreq = cinfo->dreq;  in nfs_direct_resched_write()  local
716 	spin_lock(&dreq->lock);  in nfs_direct_resched_write()
717 	dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_resched_write()
718 	spin_unlock(&dreq->lock);  in nfs_direct_resched_write()
727 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)  in nfs_direct_commit_schedule()  argument
733 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_schedule()
734 	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);  in nfs_direct_commit_schedule()
735 	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);  in nfs_direct_commit_schedule()
737 		nfs_direct_write_reschedule(dreq);  in nfs_direct_commit_schedule()
742 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);  in nfs_direct_write_schedule_work()  local
743 	int flags = dreq->flags;  in nfs_direct_write_schedule_work()
745 	dreq->flags = 0;  in nfs_direct_write_schedule_work()
748 			nfs_direct_commit_schedule(dreq);  in nfs_direct_write_schedule_work()
751 			nfs_direct_write_reschedule(dreq);  in nfs_direct_write_schedule_work()
754 			nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);  in nfs_direct_write_schedule_work()
755 			nfs_direct_complete(dreq);  in nfs_direct_write_schedule_work()
759 static void nfs_direct_write_complete(struct nfs_direct_req *dreq)  in nfs_direct_write_complete()  argument
761 	queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */  in nfs_direct_write_complete()
766 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_completion()  local
774 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_completion()
776 	spin_lock(&dreq->lock);  in nfs_direct_write_completion()
779 		dreq->error = hdr->error;  in nfs_direct_write_completion()
780 	if (dreq->error == 0) {  in nfs_direct_write_completion()
781 		nfs_direct_good_bytes(dreq, hdr);  in nfs_direct_write_completion()
783 			if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)  in nfs_direct_write_completion()
785 			else if (dreq->flags == 0) {  in nfs_direct_write_completion()
786 				nfs_direct_set_hdr_verf(dreq, hdr);  in nfs_direct_write_completion()
788 				dreq->flags = NFS_ODIRECT_DO_COMMIT;  in nfs_direct_write_completion()
789 			} else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {  in nfs_direct_write_completion()
791 				if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))  in nfs_direct_write_completion()
792 					dreq->flags =  in nfs_direct_write_completion()
797 	spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
812 	if (put_dreq(dreq))  in nfs_direct_write_completion()
813 		nfs_direct_write_complete(dreq);  in nfs_direct_write_completion()
830 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_reschedule_io()  local
832 	spin_lock(&dreq->lock);  in nfs_direct_write_reschedule_io()
833 	if (dreq->error == 0) {  in nfs_direct_write_reschedule_io()
834 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_write_reschedule_io()
839 	spin_unlock(&dreq->lock);  in nfs_direct_write_reschedule_io()
861 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_write_schedule_iovec()  argument
866 	struct inode *inode = dreq->inode;  in nfs_direct_write_schedule_iovec()
873 	desc.pg_dreq = dreq;  in nfs_direct_write_schedule_iovec()
874 	get_dreq(dreq);  in nfs_direct_write_schedule_iovec()
896 			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,  in nfs_direct_write_schedule_iovec()
903 			nfs_direct_setup_mirroring(dreq, &desc, req);  in nfs_direct_write_schedule_iovec()
922 			dreq->bytes_left -= req_len;  in nfs_direct_write_schedule_iovec()
937 		nfs_direct_req_release(dreq);  in nfs_direct_write_schedule_iovec()
941 	if (put_dreq(dreq))  in nfs_direct_write_schedule_iovec()
942 		nfs_direct_write_complete(dreq);  in nfs_direct_write_schedule_iovec()
973 	struct nfs_direct_req *dreq;  in nfs_file_direct_write()  local
992 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_write()
993 	if (!dreq)  in nfs_file_direct_write()
996 	dreq->inode = inode;  in nfs_file_direct_write()
997 	dreq->bytes_left = dreq->max_count = count;  in nfs_file_direct_write()
998 	dreq->io_start = pos;  in nfs_file_direct_write()
999 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_write()
1000 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_write()
1005 	dreq->l_ctx = l_ctx;  in nfs_file_direct_write()
1007 		dreq->iocb = iocb;  in nfs_file_direct_write()
1011 	requested = nfs_direct_write_schedule_iovec(dreq, iter, pos);  in nfs_file_direct_write()
1021 		result = nfs_direct_wait(dreq);  in nfs_file_direct_write()
1033 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()