156 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
164 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
165 		bytes = i->count;  in copy_page_to_iter_iovec()
167 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
171 	wanted = bytes;  in copy_page_to_iter_iovec()
175 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
186 		bytes -= copy;  in copy_page_to_iter_iovec()
188 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
191 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
196 			bytes -= copy;  in copy_page_to_iter_iovec()
198 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
205 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
215 	bytes -= copy;  in copy_page_to_iter_iovec()
216 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
219 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
224 		bytes -= copy;  in copy_page_to_iter_iovec()
233 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
237 	return wanted - bytes;  in copy_page_to_iter_iovec()
240 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
248 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
249 		bytes = i->count;  in copy_page_from_iter_iovec()
251 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
255 	wanted = bytes;  in copy_page_from_iter_iovec()
259 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
270 		bytes -= copy;  in copy_page_from_iter_iovec()
272 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
275 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
280 			bytes -= copy;  in copy_page_from_iter_iovec()
282 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
289 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
299 	bytes -= copy;  in copy_page_from_iter_iovec()
300 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
303 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
308 		bytes -= copy;  in copy_page_from_iter_iovec()
317 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
321 	return wanted - bytes;  in copy_page_from_iter_iovec()
367 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
375 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
376 		bytes = i->count;  in copy_page_to_iter_pipe()
378 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
390 			buf->len += bytes;  in copy_page_to_iter_pipe()
391 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
403 	buf->len = bytes;  in copy_page_to_iter_pipe()
404 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
407 	i->count -= bytes;  in copy_page_to_iter_pipe()
408 	return bytes;  in copy_page_to_iter_pipe()
418 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
426 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
540 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
550 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter()
561 	i->count -= bytes;  in copy_pipe_to_iter()
562 	return bytes;  in copy_pipe_to_iter()
572 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,  in csum_and_copy_to_pipe_iter()  argument
584 	bytes = n = push_pipe(i, bytes, &idx, &r);  in csum_and_copy_to_pipe_iter()
598 	i->count -= bytes;  in csum_and_copy_to_pipe_iter()
600 	return bytes;  in csum_and_copy_to_pipe_iter()
603 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
607 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
610 	iterate_and_advance(i, bytes, v,  in _copy_to_iter()
617 	return bytes;  in _copy_to_iter()
644 static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes,  in copy_pipe_to_iter_mcsafe()  argument
654 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter_mcsafe()
701 size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter_mcsafe()  argument
707 		return copy_pipe_to_iter_mcsafe(addr, bytes, i);  in _copy_to_iter_mcsafe()
710 	iterate_and_advance(i, bytes, v,  in _copy_to_iter_mcsafe()
717 			bytes = curr_addr - s_addr - rem;  in _copy_to_iter_mcsafe()
718 			return bytes;  in _copy_to_iter_mcsafe()
726 			bytes = curr_addr - s_addr - rem;  in _copy_to_iter_mcsafe()
727 			return bytes;  in _copy_to_iter_mcsafe()
732 	return bytes;  in _copy_to_iter_mcsafe()
737 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
746 	iterate_and_advance(i, bytes, v,  in _copy_from_iter()
753 	return bytes;  in _copy_from_iter()
757 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full()  argument
764 	if (unlikely(i->count < bytes))  in _copy_from_iter_full()
769 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full()
779 	iov_iter_advance(i, bytes);  in _copy_from_iter_full()
784 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
791 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_nocache()
799 	return bytes;  in _copy_from_iter_nocache()
818 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
825 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_flushcache()
834 	return bytes;  in _copy_from_iter_flushcache()
839 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full_nocache()  argument
846 	if (unlikely(i->count < bytes))  in _copy_from_iter_full_nocache()
848 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full_nocache()
858 	iov_iter_advance(i, bytes);  in _copy_from_iter_full_nocache()
887 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
890 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_to_iter()
894 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
898 		return bytes;  in copy_page_to_iter()
900 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
902 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in copy_page_to_iter()
906 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
909 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_from_iter()
917 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
921 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
925 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
934 	bytes = n = push_pipe(i, bytes, &idx, &off);  in pipe_zero()
945 	i->count -= bytes;  in pipe_zero()
946 	return bytes;  in pipe_zero()
949 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
952 		return pipe_zero(bytes, i);  in iov_iter_zero()
953 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
959 	return bytes;  in iov_iter_zero()
964 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
967 	if (unlikely(!page_copy_sane(page, offset, bytes))) {  in iov_iter_copy_from_user_atomic()
976 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
983 	return bytes;  in iov_iter_copy_from_user_atomic()
1404 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1415 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
1440 	return bytes;  in csum_and_copy_from_iter()
1444 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter_full()  argument
1455 	if (unlikely(i->count < bytes))  in csum_and_copy_from_iter_full()
1457 	iterate_all_kinds(i, bytes, v, ({  in csum_and_copy_from_iter_full()
1482 	iov_iter_advance(i, bytes);  in csum_and_copy_from_iter_full()
1487 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump,  in csum_and_copy_to_iter()  argument
1496 		return csum_and_copy_to_pipe_iter(addr, bytes, csum, i);  in csum_and_copy_to_iter()
1503 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
1528 	return bytes;  in csum_and_copy_to_iter()
1532 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,  in hash_and_copy_to_iter()  argument
1540 	copied = copy_to_iter(addr, bytes, i);  in hash_and_copy_to_iter()
1698 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,  in iov_iter_for_each_range()  argument
1704 	if (!bytes)  in iov_iter_for_each_range()
1707 	iterate_all_kinds(i, bytes, v, -EINVAL, ({  in iov_iter_for_each_range()