173 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
181 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
182 		bytes = i->count;  in copy_page_to_iter_iovec()
184 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
188 	wanted = bytes;  in copy_page_to_iter_iovec()
192 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
203 		bytes -= copy;  in copy_page_to_iter_iovec()
205 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
208 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
213 			bytes -= copy;  in copy_page_to_iter_iovec()
215 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
222 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
232 	bytes -= copy;  in copy_page_to_iter_iovec()
233 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
236 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
241 		bytes -= copy;  in copy_page_to_iter_iovec()
250 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
254 	return wanted - bytes;  in copy_page_to_iter_iovec()
257 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
265 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
266 		bytes = i->count;  in copy_page_from_iter_iovec()
268 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
272 	wanted = bytes;  in copy_page_from_iter_iovec()
276 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
287 		bytes -= copy;  in copy_page_from_iter_iovec()
289 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
292 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
297 			bytes -= copy;  in copy_page_from_iter_iovec()
299 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
306 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
316 	bytes -= copy;  in copy_page_from_iter_iovec()
317 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
320 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
325 		bytes -= copy;  in copy_page_from_iter_iovec()
334 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
338 	return wanted - bytes;  in copy_page_from_iter_iovec()
384 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
394 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
395 		bytes = i->count;  in copy_page_to_iter_pipe()
397 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
408 			buf->len += bytes;  in copy_page_to_iter_pipe()
409 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
422 	buf->len = bytes;  in copy_page_to_iter_pipe()
425 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
428 	i->count -= bytes;  in copy_page_to_iter_pipe()
429 	return bytes;  in copy_page_to_iter_pipe()
434  * bytes.  For each iovec, fault in each page that constitutes the iovec.
439 int iov_iter_fault_in_readable(const struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
445 		if (bytes > i->count)  in iov_iter_fault_in_readable()
446 			bytes = i->count;  in iov_iter_fault_in_readable()
447 		for (p = i->iov, skip = i->iov_offset; bytes; p++, skip = 0) {  in iov_iter_fault_in_readable()
448 			size_t len = min(bytes, p->iov_len - skip);  in iov_iter_fault_in_readable()
456 			bytes -= len;  in iov_iter_fault_in_readable()
548 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
559 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_pipe_to_iter()
572 	i->count -= bytes;  in copy_pipe_to_iter()
573 	return bytes;  in copy_pipe_to_iter()
583 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,  in csum_and_copy_to_pipe_iter()  argument
596 	bytes = push_pipe(i, bytes, &i_head, &r);  in csum_and_copy_to_pipe_iter()
597 	while (bytes) {  in csum_and_copy_to_pipe_iter()
598 		size_t chunk = min_t(size_t, bytes, PAGE_SIZE - r);  in csum_and_copy_to_pipe_iter()
604 		bytes -= chunk;  in csum_and_copy_to_pipe_iter()
614 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
617 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
620 	iterate_and_advance(i, bytes, base, len, off,  in _copy_to_iter()
625 	return bytes;  in _copy_to_iter()
639 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,  in copy_mc_pipe_to_iter()  argument
650 	n = push_pipe(i, bytes, &i_head, &off);  in copy_mc_pipe_to_iter()
674  * @bytes: total transfer length
679  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
694  * Return: number of bytes copied (may be %0)
696 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_mc_to_iter()  argument
699 		return copy_mc_pipe_to_iter(addr, bytes, i);  in _copy_mc_to_iter()
702 	__iterate_and_advance(i, bytes, base, len, off,  in _copy_mc_to_iter()
707 	return bytes;  in _copy_mc_to_iter()
712 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
720 	iterate_and_advance(i, bytes, base, len, off,  in _copy_from_iter()
725 	return bytes;  in _copy_from_iter()
729 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
735 	iterate_and_advance(i, bytes, base, len, off,  in _copy_from_iter_nocache()
740 	return bytes;  in _copy_from_iter_nocache()
748  * @bytes: total transfer length
759  * Return: number of bytes copied (may be %0)
761 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
767 	iterate_and_advance(i, bytes, base, len, off,  in _copy_from_iter_flushcache()
772 	return bytes;  in _copy_from_iter_flushcache()
801 static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in __copy_page_to_iter()  argument
805 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in __copy_page_to_iter()
808 		size_t wanted = _copy_to_iter(kaddr + offset, bytes, i);  in __copy_page_to_iter()
813 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in __copy_page_to_iter()
815 		if (unlikely(i->count < bytes))  in __copy_page_to_iter()
816 			bytes = i->count;  in __copy_page_to_iter()
817 		i->count -= bytes;  in __copy_page_to_iter()
818 		return bytes;  in __copy_page_to_iter()
824 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
828 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_to_iter()
834 				min(bytes, (size_t)PAGE_SIZE - offset), i);  in copy_page_to_iter()
836 		bytes -= n;  in copy_page_to_iter()
837 		if (!bytes || !n)  in copy_page_to_iter()
849 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
852 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_from_iter()
855 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
858 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
867 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
877 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in pipe_zero()
892 	i->count -= bytes;  in pipe_zero()
893 	return bytes;  in pipe_zero()
896 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
899 		return pipe_zero(bytes, i);  in iov_iter_zero()
900 	iterate_and_advance(i, bytes, base, len, count,  in iov_iter_zero()
905 	return bytes;  in iov_iter_zero()
909 size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes,  in copy_page_from_iter_atomic()  argument
913 	if (unlikely(!page_copy_sane(page, offset, bytes))) {  in copy_page_from_iter_atomic()
922 	iterate_and_advance(i, bytes, base, len, off,  in copy_page_from_iter_atomic()
927 	return bytes;  in copy_page_from_iter_atomic()
1181  * @count: The size of the I/O buffer in bytes.
1207  * @count: The size of the I/O buffer in bytes.
1642 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1651 	iterate_and_advance(i, bytes, base, len, off, ({  in csum_and_copy_from_iter()
1660 	return bytes;  in csum_and_copy_from_iter()
1664 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,  in csum_and_copy_to_iter()  argument
1677 		bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum);  in csum_and_copy_to_iter()
1678 	else iterate_and_advance(i, bytes, base, len, off, ({  in csum_and_copy_to_iter()
1687 	csstate->off += bytes;  in csum_and_copy_to_iter()
1688 	return bytes;  in csum_and_copy_to_iter()
1692 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,  in hash_and_copy_to_iter()  argument
1700 	copied = copy_to_iter(addr, bytes, i);  in hash_and_copy_to_iter()
1950  * Return: Negative error code on error, bytes imported on success