151 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
159 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
160 		bytes = i->count;  in copy_page_to_iter_iovec()
162 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
166 	wanted = bytes;  in copy_page_to_iter_iovec()
170 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
181 		bytes -= copy;  in copy_page_to_iter_iovec()
183 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
186 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
191 			bytes -= copy;  in copy_page_to_iter_iovec()
193 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
200 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
210 	bytes -= copy;  in copy_page_to_iter_iovec()
211 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
214 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
219 		bytes -= copy;  in copy_page_to_iter_iovec()
228 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
232 	return wanted - bytes;  in copy_page_to_iter_iovec()
235 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
243 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
244 		bytes = i->count;  in copy_page_from_iter_iovec()
246 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
250 	wanted = bytes;  in copy_page_from_iter_iovec()
254 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
265 		bytes -= copy;  in copy_page_from_iter_iovec()
267 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
270 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
275 			bytes -= copy;  in copy_page_from_iter_iovec()
277 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
284 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
294 	bytes -= copy;  in copy_page_from_iter_iovec()
295 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
298 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
303 		bytes -= copy;  in copy_page_from_iter_iovec()
312 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
316 	return wanted - bytes;  in copy_page_from_iter_iovec()
362 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
370 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
371 		bytes = i->count;  in copy_page_to_iter_pipe()
373 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
385 			buf->len += bytes;  in copy_page_to_iter_pipe()
386 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
398 	buf->len = bytes;  in copy_page_to_iter_pipe()
399 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
402 	i->count -= bytes;  in copy_page_to_iter_pipe()
403 	return bytes;  in copy_page_to_iter_pipe()
413 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
421 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
533 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
543 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter()
554 	i->count -= bytes;  in copy_pipe_to_iter()
555 	return bytes;  in copy_pipe_to_iter()
558 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
562 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
565 	iterate_and_advance(i, bytes, v,  in _copy_to_iter()
572 	return bytes;  in _copy_to_iter()
599 static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes,  in copy_pipe_to_iter_mcsafe()  argument
609 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter_mcsafe()
656 size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter_mcsafe()  argument
662 		return copy_pipe_to_iter_mcsafe(addr, bytes, i);  in _copy_to_iter_mcsafe()
665 	iterate_and_advance(i, bytes, v,  in _copy_to_iter_mcsafe()
672 			bytes = curr_addr - s_addr - rem;  in _copy_to_iter_mcsafe()
673 			return bytes;  in _copy_to_iter_mcsafe()
681 			bytes = curr_addr - s_addr - rem;  in _copy_to_iter_mcsafe()
682 			return bytes;  in _copy_to_iter_mcsafe()
687 	return bytes;  in _copy_to_iter_mcsafe()
692 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
701 	iterate_and_advance(i, bytes, v,  in _copy_from_iter()
708 	return bytes;  in _copy_from_iter()
712 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full()  argument
719 	if (unlikely(i->count < bytes))  in _copy_from_iter_full()
724 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full()
734 	iov_iter_advance(i, bytes);  in _copy_from_iter_full()
739 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
746 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_nocache()
754 	return bytes;  in _copy_from_iter_nocache()
773 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
780 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_flushcache()
789 	return bytes;  in _copy_from_iter_flushcache()
794 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full_nocache()  argument
801 	if (unlikely(i->count < bytes))  in _copy_from_iter_full_nocache()
803 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full_nocache()
813 	iov_iter_advance(i, bytes);  in _copy_from_iter_full_nocache()
829 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
832 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_to_iter()
836 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
840 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
842 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in copy_page_to_iter()
846 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
849 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_from_iter()
857 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
861 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
865 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
874 	bytes = n = push_pipe(i, bytes, &idx, &off);  in pipe_zero()
885 	i->count -= bytes;  in pipe_zero()
886 	return bytes;  in pipe_zero()
889 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
892 		return pipe_zero(bytes, i);  in iov_iter_zero()
893 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
899 	return bytes;  in iov_iter_zero()
904 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
907 	if (unlikely(!page_copy_sane(page, offset, bytes))) {  in iov_iter_copy_from_user_atomic()
916 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
923 	return bytes;  in iov_iter_copy_from_user_atomic()
1309 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1320 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
1347 	return bytes;  in csum_and_copy_from_iter()
1351 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter_full()  argument
1362 	if (unlikely(i->count < bytes))  in csum_and_copy_from_iter_full()
1364 	iterate_all_kinds(i, bytes, v, ({  in csum_and_copy_from_iter_full()
1391 	iov_iter_advance(i, bytes);  in csum_and_copy_from_iter_full()
1396 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_to_iter()  argument
1407 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
1434 	return bytes;  in csum_and_copy_to_iter()
1580 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,  in iov_iter_for_each_range()  argument
1586 	if (!bytes)  in iov_iter_for_each_range()
1589 	iterate_all_kinds(i, bytes, v, -EINVAL, ({  in iov_iter_for_each_range()