3691 	struct cifs_readdata *rdata;  in cifs_readdata_direct_alloc()  local
3693 	rdata = kzalloc(sizeof(*rdata), GFP_KERNEL);  in cifs_readdata_direct_alloc()
3694 	if (rdata != NULL) {  in cifs_readdata_direct_alloc()
3695 		rdata->pages = pages;  in cifs_readdata_direct_alloc()
3696 		kref_init(&rdata->refcount);  in cifs_readdata_direct_alloc()
3697 		INIT_LIST_HEAD(&rdata->list);  in cifs_readdata_direct_alloc()
3698 		init_completion(&rdata->done);  in cifs_readdata_direct_alloc()
3699 		INIT_WORK(&rdata->work, complete);  in cifs_readdata_direct_alloc()
3702 	return rdata;  in cifs_readdata_direct_alloc()
3724 	struct cifs_readdata *rdata = container_of(refcount,  in cifs_readdata_release()  local
3727 	if (rdata->mr) {  in cifs_readdata_release()
3728 		smbd_deregister_mr(rdata->mr);  in cifs_readdata_release()
3729 		rdata->mr = NULL;  in cifs_readdata_release()
3732 	if (rdata->cfile)  in cifs_readdata_release()
3733 		cifsFileInfo_put(rdata->cfile);  in cifs_readdata_release()
3735 	kvfree(rdata->pages);  in cifs_readdata_release()
3736 	kfree(rdata);  in cifs_readdata_release()
3740 cifs_read_allocate_pages(struct cifs_readdata *rdata, unsigned int nr_pages)  in cifs_read_allocate_pages()  argument
3752 		rdata->pages[i] = page;  in cifs_read_allocate_pages()
3759 			put_page(rdata->pages[i]);  in cifs_read_allocate_pages()
3760 			rdata->pages[i] = NULL;  in cifs_read_allocate_pages()
3769 	struct cifs_readdata *rdata = container_of(refcount,  in cifs_uncached_readdata_release()  local
3773 	kref_put(&rdata->ctx->refcount, cifs_aio_ctx_release);  in cifs_uncached_readdata_release()
3774 	for (i = 0; i < rdata->nr_pages; i++) {  in cifs_uncached_readdata_release()
3775 		put_page(rdata->pages[i]);  in cifs_uncached_readdata_release()
3790 cifs_readdata_to_iov(struct cifs_readdata *rdata, struct iov_iter *iter)  in cifs_readdata_to_iov()  argument
3792 	size_t remaining = rdata->got_bytes;  in cifs_readdata_to_iov()
3795 	for (i = 0; i < rdata->nr_pages; i++) {  in cifs_readdata_to_iov()
3796 		struct page *page = rdata->pages[i];  in cifs_readdata_to_iov()
3819 	struct cifs_readdata *rdata = container_of(work,  in cifs_uncached_readv_complete()  local
3822 	complete(&rdata->done);  in cifs_uncached_readv_complete()
3823 	collect_uncached_read_data(rdata->ctx);  in cifs_uncached_readv_complete()
3825 	kref_put(&rdata->refcount, cifs_uncached_readdata_release);  in cifs_uncached_readv_complete()
3830 		    struct cifs_readdata *rdata, struct iov_iter *iter,  in uncached_fill_pages()  argument
3835 	unsigned int nr_pages = rdata->nr_pages;  in uncached_fill_pages()
3836 	unsigned int page_offset = rdata->page_offset;  in uncached_fill_pages()
3838 	rdata->got_bytes = 0;  in uncached_fill_pages()
3839 	rdata->tailsz = PAGE_SIZE;  in uncached_fill_pages()
3841 		struct page *page = rdata->pages[i];  in uncached_fill_pages()
3843 		unsigned int segment_size = rdata->pagesz;  in uncached_fill_pages()
3853 			rdata->pages[i] = NULL;  in uncached_fill_pages()
3854 			rdata->nr_pages--;  in uncached_fill_pages()
3864 			rdata->tailsz = len;  in uncached_fill_pages()
3871 		else if (rdata->mr)  in uncached_fill_pages()
3880 		rdata->got_bytes += result;  in uncached_fill_pages()
3883 	return rdata->got_bytes > 0 && result != -ECONNABORTED ?  in uncached_fill_pages()
3884 						rdata->got_bytes : result;  in uncached_fill_pages()
3889 			      struct cifs_readdata *rdata, unsigned int len)  in cifs_uncached_read_into_pages()  argument
3891 	return uncached_fill_pages(server, rdata, NULL, len);  in cifs_uncached_read_into_pages()
3896 			      struct cifs_readdata *rdata,  in cifs_uncached_copy_into_pages()  argument
3899 	return uncached_fill_pages(server, rdata, iter, iter->count);  in cifs_uncached_copy_into_pages()
3902 static int cifs_resend_rdata(struct cifs_readdata *rdata,  in cifs_resend_rdata()  argument
3912 	server = rdata->server;  in cifs_resend_rdata()
3915 		if (rdata->cfile->invalidHandle) {  in cifs_resend_rdata()
3916 			rc = cifs_reopen_file(rdata->cfile, true);  in cifs_resend_rdata()
3929 			rc = server->ops->wait_mtu_credits(server, rdata->bytes,  in cifs_resend_rdata()
3935 			if (rsize < rdata->bytes) {  in cifs_resend_rdata()
3939 		} while (rsize < rdata->bytes);  in cifs_resend_rdata()
3940 		rdata->credits = credits;  in cifs_resend_rdata()
3942 		rc = adjust_credits(server, &rdata->credits, rdata->bytes);  in cifs_resend_rdata()
3944 			if (rdata->cfile->invalidHandle)  in cifs_resend_rdata()
3948 				if (rdata->mr) {  in cifs_resend_rdata()
3949 					rdata->mr->need_invalidate = true;  in cifs_resend_rdata()
3950 					smbd_deregister_mr(rdata->mr);  in cifs_resend_rdata()
3951 					rdata->mr = NULL;  in cifs_resend_rdata()
3954 				rc = server->ops->async_readv(rdata);  in cifs_resend_rdata()
3961 			list_add_tail(&rdata->list, rdata_list);  in cifs_resend_rdata()
3966 		add_credits_and_wake_if(server, &rdata->credits, 0);  in cifs_resend_rdata()
3970 	kref_put(&rdata->refcount, cifs_uncached_readdata_release);  in cifs_resend_rdata()
3979 	struct cifs_readdata *rdata;  in cifs_send_async_read()  local
4042 			rdata = cifs_readdata_direct_alloc(  in cifs_send_async_read()
4044 			if (!rdata) {  in cifs_send_async_read()
4051 			rdata->page_offset = start;  in cifs_send_async_read()
4052 			rdata->tailsz = npages > 1 ?  in cifs_send_async_read()
4060 			rdata = cifs_readdata_alloc(npages,  in cifs_send_async_read()
4062 			if (!rdata) {  in cifs_send_async_read()
4068 			rc = cifs_read_allocate_pages(rdata, npages);  in cifs_send_async_read()
4070 				kvfree(rdata->pages);  in cifs_send_async_read()
4071 				kfree(rdata);  in cifs_send_async_read()
4076 			rdata->tailsz = PAGE_SIZE;  in cifs_send_async_read()
4079 		rdata->server = server;  in cifs_send_async_read()
4080 		rdata->cfile = cifsFileInfo_get(open_file);  in cifs_send_async_read()
4081 		rdata->nr_pages = npages;  in cifs_send_async_read()
4082 		rdata->offset = offset;  in cifs_send_async_read()
4083 		rdata->bytes = cur_len;  in cifs_send_async_read()
4084 		rdata->pid = pid;  in cifs_send_async_read()
4085 		rdata->pagesz = PAGE_SIZE;  in cifs_send_async_read()
4086 		rdata->read_into_pages = cifs_uncached_read_into_pages;  in cifs_send_async_read()
4087 		rdata->copy_into_pages = cifs_uncached_copy_into_pages;  in cifs_send_async_read()
4088 		rdata->credits = credits_on_stack;  in cifs_send_async_read()
4089 		rdata->ctx = ctx;  in cifs_send_async_read()
4092 		rc = adjust_credits(server, &rdata->credits, rdata->bytes);  in cifs_send_async_read()
4095 			if (rdata->cfile->invalidHandle)  in cifs_send_async_read()
4098 				rc = server->ops->async_readv(rdata);  in cifs_send_async_read()
4102 			add_credits_and_wake_if(server, &rdata->credits, 0);  in cifs_send_async_read()
4103 			kref_put(&rdata->refcount,  in cifs_send_async_read()
4112 		list_add_tail(&rdata->list, rdata_list);  in cifs_send_async_read()
4123 	struct cifs_readdata *rdata, *tmp;  in collect_uncached_read_data()  local
4140 	list_for_each_entry_safe(rdata, tmp, &ctx->list, list) {  in collect_uncached_read_data()
4142 			if (!try_wait_for_completion(&rdata->done)) {  in collect_uncached_read_data()
4147 			if (rdata->result == -EAGAIN) {  in collect_uncached_read_data()
4150 				unsigned int got_bytes = rdata->got_bytes;  in collect_uncached_read_data()
4152 				list_del_init(&rdata->list);  in collect_uncached_read_data()
4160 				if (got_bytes && got_bytes < rdata->bytes) {  in collect_uncached_read_data()
4163 						rc = cifs_readdata_to_iov(rdata, to);  in collect_uncached_read_data()
4165 						kref_put(&rdata->refcount,  in collect_uncached_read_data()
4177 						rdata,  in collect_uncached_read_data()
4181 						rdata->offset + got_bytes,  in collect_uncached_read_data()
4182 						rdata->bytes - got_bytes,  in collect_uncached_read_data()
4183 						rdata->cfile, cifs_sb,  in collect_uncached_read_data()
4186 					kref_put(&rdata->refcount,  in collect_uncached_read_data()
4193 			} else if (rdata->result)  in collect_uncached_read_data()
4194 				rc = rdata->result;  in collect_uncached_read_data()
4196 				rc = cifs_readdata_to_iov(rdata, to);  in collect_uncached_read_data()
4199 			if (rdata->got_bytes && rdata->got_bytes < rdata->bytes)  in collect_uncached_read_data()
4202 			ctx->total_len += rdata->got_bytes;  in collect_uncached_read_data()
4204 		list_del_init(&rdata->list);  in collect_uncached_read_data()
4205 		kref_put(&rdata->refcount, cifs_uncached_readdata_release);  in collect_uncached_read_data()
4551 	struct cifs_readdata *rdata = container_of(work,  in cifs_readv_complete()  local
4554 	got_bytes = rdata->got_bytes;  in cifs_readv_complete()
4555 	for (i = 0; i < rdata->nr_pages; i++) {  in cifs_readv_complete()
4556 		struct page *page = rdata->pages[i];  in cifs_readv_complete()
4558 		if (rdata->result == 0 ||  in cifs_readv_complete()
4559 		    (rdata->result == -EAGAIN && got_bytes)) {  in cifs_readv_complete()
4565 		if (rdata->result == 0 ||  in cifs_readv_complete()
4566 		    (rdata->result == -EAGAIN && got_bytes))  in cifs_readv_complete()
4567 			cifs_readpage_to_fscache(rdata->mapping->host, page);  in cifs_readv_complete()
4574 		rdata->pages[i] = NULL;  in cifs_readv_complete()
4576 	kref_put(&rdata->refcount, cifs_readdata_release);  in cifs_readv_complete()
4581 		     struct cifs_readdata *rdata, struct iov_iter *iter,  in readpages_fill_pages()  argument
4588 	unsigned int nr_pages = rdata->nr_pages;  in readpages_fill_pages()
4589 	unsigned int page_offset = rdata->page_offset;  in readpages_fill_pages()
4592 	eof = CIFS_I(rdata->mapping->host)->server_eof;  in readpages_fill_pages()
4596 	rdata->got_bytes = 0;  in readpages_fill_pages()
4597 	rdata->tailsz = PAGE_SIZE;  in readpages_fill_pages()
4599 		struct page *page = rdata->pages[i];  in readpages_fill_pages()
4600 		unsigned int to_read = rdata->pagesz;  in readpages_fill_pages()
4615 			n = rdata->tailsz = len;  in readpages_fill_pages()
4631 			rdata->pages[i] = NULL;  in readpages_fill_pages()
4632 			rdata->nr_pages--;  in readpages_fill_pages()
4638 			rdata->pages[i] = NULL;  in readpages_fill_pages()
4639 			rdata->nr_pages--;  in readpages_fill_pages()
4647 		else if (rdata->mr)  in readpages_fill_pages()
4656 		rdata->got_bytes += result;  in readpages_fill_pages()
4659 	return rdata->got_bytes > 0 && result != -ECONNABORTED ?  in readpages_fill_pages()
4660 						rdata->got_bytes : result;  in readpages_fill_pages()
4665 			       struct cifs_readdata *rdata, unsigned int len)  in cifs_readpages_read_into_pages()  argument
4667 	return readpages_fill_pages(server, rdata, NULL, len);  in cifs_readpages_read_into_pages()
4672 			       struct cifs_readdata *rdata,  in cifs_readpages_copy_into_pages()  argument
4675 	return readpages_fill_pages(server, rdata, iter, iter->count);  in cifs_readpages_copy_into_pages()
4710 		struct cifs_readdata *rdata;  in cifs_readahead()  local
4787 		rdata = cifs_readdata_alloc(nr_pages, cifs_readv_complete);  in cifs_readahead()
4788 		if (!rdata) {  in cifs_readahead()
4794 		got = __readahead_batch(ractl, rdata->pages, nr_pages);  in cifs_readahead()
4801 		rdata->nr_pages = nr_pages;  in cifs_readahead()
4802 		rdata->bytes	= readahead_batch_length(ractl);  in cifs_readahead()
4803 		rdata->cfile	= cifsFileInfo_get(open_file);  in cifs_readahead()
4804 		rdata->server	= server;  in cifs_readahead()
4805 		rdata->mapping	= ractl->mapping;  in cifs_readahead()
4806 		rdata->offset	= readahead_pos(ractl);  in cifs_readahead()
4807 		rdata->pid	= pid;  in cifs_readahead()
4808 		rdata->pagesz	= PAGE_SIZE;  in cifs_readahead()
4809 		rdata->tailsz	= PAGE_SIZE;  in cifs_readahead()
4810 		rdata->read_into_pages = cifs_readpages_read_into_pages;  in cifs_readahead()
4811 		rdata->copy_into_pages = cifs_readpages_copy_into_pages;  in cifs_readahead()
4812 		rdata->credits	= credits_on_stack;  in cifs_readahead()
4814 		rc = adjust_credits(server, &rdata->credits, rdata->bytes);  in cifs_readahead()
4816 			if (rdata->cfile->invalidHandle)  in cifs_readahead()
4819 				rc = server->ops->async_readv(rdata);  in cifs_readahead()
4823 			add_credits_and_wake_if(server, &rdata->credits, 0);  in cifs_readahead()
4824 			for (i = 0; i < rdata->nr_pages; i++) {  in cifs_readahead()
4825 				page = rdata->pages[i];  in cifs_readahead()
4830 			kref_put(&rdata->refcount, cifs_readdata_release);  in cifs_readahead()
4834 		kref_put(&rdata->refcount, cifs_readdata_release);  in cifs_readahead()