20 #define iterate_iovec(i, n, base, len, off, __p, STEP) {	\  argument
22 	size_t skip = i->iov_offset;				\
37 	i->iov_offset = skip;					\
41 #define iterate_bvec(i, n, base, len, off, p, STEP) {		\  argument
43 	unsigned skip = i->iov_offset;				\
65 	i->iov_offset = skip;					\
69 #define iterate_xarray(i, n, base, len, __off, STEP) {		\  argument
73 	loff_t start = i->xarray_start + i->iov_offset;		\
78 	XA_STATE(xas, i->xarray, index);			\
107 	i->iov_offset += __off;						\
111 #define __iterate_and_advance(i, n, base, len, off, I, K) {	\  argument
112 	if (unlikely(i->count < n))				\
113 		n = i->count;					\
115 		if (likely(iter_is_iovec(i))) {			\
116 			const struct iovec *iov = i->iov;	\
119 			iterate_iovec(i, n, base, len, off,	\
120 						iov, (I))	\
121 			i->nr_segs -= iov - i->iov;		\
122 			i->iov = iov;				\
123 		} else if (iov_iter_is_bvec(i)) {		\
124 			const struct bio_vec *bvec = i->bvec;	\
127 			iterate_bvec(i, n, base, len, off,	\
129 			i->nr_segs -= bvec - i->bvec;		\
130 			i->bvec = bvec;				\
131 		} else if (iov_iter_is_kvec(i)) {		\
132 			const struct kvec *kvec = i->kvec;	\
135 			iterate_iovec(i, n, base, len, off,	\
137 			i->nr_segs -= kvec - i->kvec;		\
138 			i->kvec = kvec;				\
139 		} else if (iov_iter_is_xarray(i)) {		\
142 			iterate_xarray(i, n, base, len, off,	\
145 		i->count -= n;					\
148 #define iterate_and_advance(i, n, base, len, off, I, K) \  argument
149 	__iterate_and_advance(i, n, base, len, off, I, ((void)(K),0))
174 			 struct iov_iter *i)  in copy_page_to_iter_iovec()  argument
181 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
182 		bytes = i->count;  in copy_page_to_iter_iovec()
189 	iov = i->iov;  in copy_page_to_iter_iovec()
190 	skip = i->iov_offset;  in copy_page_to_iter_iovec()
250 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
251 	i->nr_segs -= iov - i->iov;  in copy_page_to_iter_iovec()
252 	i->iov = iov;  in copy_page_to_iter_iovec()
253 	i->iov_offset = skip;  in copy_page_to_iter_iovec()
258 			 struct iov_iter *i)  in copy_page_from_iter_iovec()  argument
265 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
266 		bytes = i->count;  in copy_page_from_iter_iovec()
273 	iov = i->iov;  in copy_page_from_iter_iovec()
274 	skip = i->iov_offset;  in copy_page_from_iter_iovec()
334 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
335 	i->nr_segs -= iov - i->iov;  in copy_page_from_iter_iovec()
336 	i->iov = iov;  in copy_page_from_iter_iovec()
337 	i->iov_offset = skip;  in copy_page_from_iter_iovec()
342 static bool sanity(const struct iov_iter *i)  in sanity()  argument
344 	struct pipe_inode_info *pipe = i->pipe;  in sanity()
349 	unsigned int i_head = i->head;  in sanity()
352 	if (i->iov_offset) {  in sanity()
360 		if (unlikely(p->offset + p->len != i->iov_offset))  in sanity()
368 	printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);  in sanity()
381 #define sanity(i) true  argument
385 			 struct iov_iter *i)  in copy_page_to_iter_pipe()  argument
387 	struct pipe_inode_info *pipe = i->pipe;  in copy_page_to_iter_pipe()
391 	unsigned int i_head = i->head;  in copy_page_to_iter_pipe()
394 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
395 		bytes = i->count;  in copy_page_to_iter_pipe()
400 	if (!sanity(i))  in copy_page_to_iter_pipe()
403 	off = i->iov_offset;  in copy_page_to_iter_pipe()
409 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
425 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
426 	i->head = i_head;  in copy_page_to_iter_pipe()
428 	i->count -= bytes;  in copy_page_to_iter_pipe()
436  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
439 int iov_iter_fault_in_readable(const struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
441 	if (iter_is_iovec(i)) {  in iov_iter_fault_in_readable()
445 		if (bytes > i->count)  in iov_iter_fault_in_readable()
446 			bytes = i->count;  in iov_iter_fault_in_readable()
447 		for (p = i->iov, skip = i->iov_offset; bytes; p++, skip = 0) {  in iov_iter_fault_in_readable()
463 void iov_iter_init(struct iov_iter *i, unsigned int direction,  in iov_iter_init()  argument
468 	*i = (struct iov_iter) {  in iov_iter_init()
484 static inline void data_start(const struct iov_iter *i,  in data_start()  argument
487 	unsigned int p_mask = i->pipe->ring_size - 1;  in data_start()
488 	unsigned int iter_head = i->head;  in data_start()
489 	size_t off = i->iov_offset;  in data_start()
491 	if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||  in data_start()
500 static size_t push_pipe(struct iov_iter *i, size_t size,  in push_pipe()  argument
503 	struct pipe_inode_info *pipe = i->pipe;  in push_pipe()
510 	if (unlikely(size > i->count))  in push_pipe()
511 		size = i->count;  in push_pipe()
516 	data_start(i, &iter_head, &off);  in push_pipe()
549 				struct iov_iter *i)  in copy_pipe_to_iter()  argument
551 	struct pipe_inode_info *pipe = i->pipe;  in copy_pipe_to_iter()
556 	if (!sanity(i))  in copy_pipe_to_iter()
559 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_pipe_to_iter()
565 		i->head = i_head;  in copy_pipe_to_iter()
566 		i->iov_offset = off + chunk;  in copy_pipe_to_iter()
572 	i->count -= bytes;  in copy_pipe_to_iter()
584 					 struct iov_iter *i, __wsum *sump)  in csum_and_copy_to_pipe_iter()  argument
586 	struct pipe_inode_info *pipe = i->pipe;  in csum_and_copy_to_pipe_iter()
593 	if (!sanity(i))  in csum_and_copy_to_pipe_iter()
596 	bytes = push_pipe(i, bytes, &i_head, &r);  in csum_and_copy_to_pipe_iter()
602 		i->head = i_head;  in csum_and_copy_to_pipe_iter()
603 		i->iov_offset = r + chunk;  in csum_and_copy_to_pipe_iter()
610 	i->count -= off;  in csum_and_copy_to_pipe_iter()
614 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
616 	if (unlikely(iov_iter_is_pipe(i)))  in _copy_to_iter()
617 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
618 	if (iter_is_iovec(i))  in _copy_to_iter()
620 	iterate_and_advance(i, bytes, base, len, off,  in _copy_to_iter()
640 				struct iov_iter *i)  in copy_mc_pipe_to_iter()  argument
642 	struct pipe_inode_info *pipe = i->pipe;  in copy_mc_pipe_to_iter()
647 	if (!sanity(i))  in copy_mc_pipe_to_iter()
650 	n = push_pipe(i, bytes, &i_head, &off);  in copy_mc_pipe_to_iter()
658 		i->head = i_head;  in copy_mc_pipe_to_iter()
659 		i->iov_offset = off + chunk;  in copy_mc_pipe_to_iter()
667 	i->count -= xfer;  in copy_mc_pipe_to_iter()
675  * @i: destination iterator
696 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_mc_to_iter()  argument
698 	if (unlikely(iov_iter_is_pipe(i)))  in _copy_mc_to_iter()
699 		return copy_mc_pipe_to_iter(addr, bytes, i);  in _copy_mc_to_iter()
700 	if (iter_is_iovec(i))  in _copy_mc_to_iter()
702 	__iterate_and_advance(i, bytes, base, len, off,  in _copy_mc_to_iter()
712 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
714 	if (unlikely(iov_iter_is_pipe(i))) {  in _copy_from_iter()
718 	if (iter_is_iovec(i))  in _copy_from_iter()
720 	iterate_and_advance(i, bytes, base, len, off,  in _copy_from_iter()
729 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
731 	if (unlikely(iov_iter_is_pipe(i))) {  in _copy_from_iter_nocache()
735 	iterate_and_advance(i, bytes, base, len, off,  in _copy_from_iter_nocache()
749  * @i: source iterator
761 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
763 	if (unlikely(iov_iter_is_pipe(i))) {  in _copy_from_iter_flushcache()
767 	iterate_and_advance(i, bytes, base, len, off,  in _copy_from_iter_flushcache()
802 			 struct iov_iter *i)  in __copy_page_to_iter()  argument
804 	if (likely(iter_is_iovec(i)))  in __copy_page_to_iter()
805 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in __copy_page_to_iter()
806 	if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) {  in __copy_page_to_iter()
808 		size_t wanted = _copy_to_iter(kaddr + offset, bytes, i);  in __copy_page_to_iter()
812 	if (iov_iter_is_pipe(i))  in __copy_page_to_iter()
813 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in __copy_page_to_iter()
814 	if (unlikely(iov_iter_is_discard(i))) {  in __copy_page_to_iter()
815 		if (unlikely(i->count < bytes))  in __copy_page_to_iter()
816 			bytes = i->count;  in __copy_page_to_iter()
817 		i->count -= bytes;  in __copy_page_to_iter()
825 			 struct iov_iter *i)  in copy_page_to_iter()  argument
834 				min(bytes, (size_t)PAGE_SIZE - offset), i);  in copy_page_to_iter()
850 			 struct iov_iter *i)  in copy_page_from_iter()  argument
854 	if (likely(iter_is_iovec(i)))  in copy_page_from_iter()
855 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
856 	if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) {  in copy_page_from_iter()
858 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
867 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
869 	struct pipe_inode_info *pipe = i->pipe;  in pipe_zero()
874 	if (!sanity(i))  in pipe_zero()
877 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in pipe_zero()
886 		i->head = i_head;  in pipe_zero()
887 		i->iov_offset = off + chunk;  in pipe_zero()
892 	i->count -= bytes;  in pipe_zero()
896 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
898 	if (unlikely(iov_iter_is_pipe(i)))  in iov_iter_zero()
899 		return pipe_zero(bytes, i);  in iov_iter_zero()
900 	iterate_and_advance(i, bytes, base, len, count,  in iov_iter_zero()
910 				  struct iov_iter *i)  in copy_page_from_iter_atomic()  argument
917 	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {  in copy_page_from_iter_atomic()
922 	iterate_and_advance(i, bytes, base, len, off,  in copy_page_from_iter_atomic()
931 static inline void pipe_truncate(struct iov_iter *i)  in pipe_truncate()  argument
933 	struct pipe_inode_info *pipe = i->pipe;  in pipe_truncate()
940 		unsigned int i_head = i->head;  in pipe_truncate()
941 		size_t off = i->iov_offset;  in pipe_truncate()
957 static void pipe_advance(struct iov_iter *i, size_t size)  in pipe_advance()  argument
959 	struct pipe_inode_info *pipe = i->pipe;  in pipe_advance()
963 		unsigned int i_head = i->head;  in pipe_advance()
964 		size_t off = i->iov_offset, left = size;  in pipe_advance()
975 		i->head = i_head;  in pipe_advance()
976 		i->iov_offset = buf->offset + left;  in pipe_advance()
978 	i->count -= size;  in pipe_advance()
980 	pipe_truncate(i);  in pipe_advance()
983 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)  in iov_iter_bvec_advance()  argument
987 	bi.bi_size = i->count;  in iov_iter_bvec_advance()
988 	bi.bi_bvec_done = i->iov_offset;  in iov_iter_bvec_advance()
990 	bvec_iter_advance(i->bvec, &bi, size);  in iov_iter_bvec_advance()
992 	i->bvec += bi.bi_idx;  in iov_iter_bvec_advance()
993 	i->nr_segs -= bi.bi_idx;  in iov_iter_bvec_advance()
994 	i->count = bi.bi_size;  in iov_iter_bvec_advance()
995 	i->iov_offset = bi.bi_bvec_done;  in iov_iter_bvec_advance()
998 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)  in iov_iter_iovec_advance()  argument
1002 	if (!i->count)  in iov_iter_iovec_advance()
1004 	i->count -= size;  in iov_iter_iovec_advance()
1006 	size += i->iov_offset; // from beginning of current segment  in iov_iter_iovec_advance()
1007 	for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) {  in iov_iter_iovec_advance()
1012 	i->iov_offset = size;  in iov_iter_iovec_advance()
1013 	i->nr_segs -= iov - i->iov;  in iov_iter_iovec_advance()
1014 	i->iov = iov;  in iov_iter_iovec_advance()
1017 void iov_iter_advance(struct iov_iter *i, size_t size)  in iov_iter_advance()  argument
1019 	if (unlikely(i->count < size))  in iov_iter_advance()
1020 		size = i->count;  in iov_iter_advance()
1021 	if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) {  in iov_iter_advance()
1023 		iov_iter_iovec_advance(i, size);  in iov_iter_advance()
1024 	} else if (iov_iter_is_bvec(i)) {  in iov_iter_advance()
1025 		iov_iter_bvec_advance(i, size);  in iov_iter_advance()
1026 	} else if (iov_iter_is_pipe(i)) {  in iov_iter_advance()
1027 		pipe_advance(i, size);  in iov_iter_advance()
1028 	} else if (unlikely(iov_iter_is_xarray(i))) {  in iov_iter_advance()
1029 		i->iov_offset += size;  in iov_iter_advance()
1030 		i->count -= size;  in iov_iter_advance()
1031 	} else if (iov_iter_is_discard(i)) {  in iov_iter_advance()
1032 		i->count -= size;  in iov_iter_advance()
1037 void iov_iter_revert(struct iov_iter *i, size_t unroll)  in iov_iter_revert()  argument
1043 	i->count += unroll;  in iov_iter_revert()
1044 	if (unlikely(iov_iter_is_pipe(i))) {  in iov_iter_revert()
1045 		struct pipe_inode_info *pipe = i->pipe;  in iov_iter_revert()
1047 		unsigned int i_head = i->head;  in iov_iter_revert()
1048 		size_t off = i->iov_offset;  in iov_iter_revert()
1057 			if (!unroll && i_head == i->start_head) {  in iov_iter_revert()
1065 		i->iov_offset = off;  in iov_iter_revert()
1066 		i->head = i_head;  in iov_iter_revert()
1067 		pipe_truncate(i);  in iov_iter_revert()
1070 	if (unlikely(iov_iter_is_discard(i)))  in iov_iter_revert()
1072 	if (unroll <= i->iov_offset) {  in iov_iter_revert()
1073 		i->iov_offset -= unroll;  in iov_iter_revert()
1076 	unroll -= i->iov_offset;  in iov_iter_revert()
1077 	if (iov_iter_is_xarray(i)) {  in iov_iter_revert()
1082 	} else if (iov_iter_is_bvec(i)) {  in iov_iter_revert()
1083 		const struct bio_vec *bvec = i->bvec;  in iov_iter_revert()
1086 			i->nr_segs++;  in iov_iter_revert()
1088 				i->bvec = bvec;  in iov_iter_revert()
1089 				i->iov_offset = n - unroll;  in iov_iter_revert()
1095 		const struct iovec *iov = i->iov;  in iov_iter_revert()
1098 			i->nr_segs++;  in iov_iter_revert()
1100 				i->iov = iov;  in iov_iter_revert()
1101 				i->iov_offset = n - unroll;  in iov_iter_revert()
1113 size_t iov_iter_single_seg_count(const struct iov_iter *i)  in iov_iter_single_seg_count()  argument
1115 	if (i->nr_segs > 1) {  in iov_iter_single_seg_count()
1116 		if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))  in iov_iter_single_seg_count()
1117 			return min(i->count, i->iov->iov_len - i->iov_offset);  in iov_iter_single_seg_count()
1118 		if (iov_iter_is_bvec(i))  in iov_iter_single_seg_count()
1119 			return min(i->count, i->bvec->bv_len - i->iov_offset);  in iov_iter_single_seg_count()
1121 	return i->count;  in iov_iter_single_seg_count()
1125 void iov_iter_kvec(struct iov_iter *i, unsigned int direction,  in iov_iter_kvec()  argument
1130 	*i = (struct iov_iter){  in iov_iter_kvec()
1141 void iov_iter_bvec(struct iov_iter *i, unsigned int direction,  in iov_iter_bvec()  argument
1146 	*i = (struct iov_iter){  in iov_iter_bvec()
1157 void iov_iter_pipe(struct iov_iter *i, unsigned int direction,  in iov_iter_pipe()  argument
1163 	*i = (struct iov_iter){  in iov_iter_pipe()
1176  * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
1177  * @i: The iterator to initialise.
1181  * @count: The size of the I/O buffer in bytes.
1183  * Set up an I/O iterator to either draw data out of the pages attached to an
1188 void iov_iter_xarray(struct iov_iter *i, unsigned int direction,  in iov_iter_xarray()  argument
1192 	*i = (struct iov_iter) {  in iov_iter_xarray()
1204  * iov_iter_discard - Initialise an I/O iterator that discards data
1205  * @i: The iterator to initialise.
1207  * @count: The size of the I/O buffer in bytes.
1209  * Set up an I/O iterator that just discards everything that's written to it.
1212 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)  in iov_iter_discard()  argument
1215 	*i = (struct iov_iter){  in iov_iter_discard()
1224 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)  in iov_iter_alignment_iovec()  argument
1227 	size_t size = i->count;  in iov_iter_alignment_iovec()
1228 	size_t skip = i->iov_offset;  in iov_iter_alignment_iovec()
1231 	for (k = 0; k < i->nr_segs; k++, skip = 0) {  in iov_iter_alignment_iovec()
1232 		size_t len = i->iov[k].iov_len - skip;  in iov_iter_alignment_iovec()
1234 			res |= (unsigned long)i->iov[k].iov_base + skip;  in iov_iter_alignment_iovec()
1246 static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i)  in iov_iter_alignment_bvec()  argument
1249 	size_t size = i->count;  in iov_iter_alignment_bvec()
1250 	unsigned skip = i->iov_offset;  in iov_iter_alignment_bvec()
1253 	for (k = 0; k < i->nr_segs; k++, skip = 0) {  in iov_iter_alignment_bvec()
1254 		size_t len = i->bvec[k].bv_len - skip;  in iov_iter_alignment_bvec()
1255 		res |= (unsigned long)i->bvec[k].bv_offset + skip;  in iov_iter_alignment_bvec()
1266 unsigned long iov_iter_alignment(const struct iov_iter *i)  in iov_iter_alignment()  argument
1269 	if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))  in iov_iter_alignment()
1270 		return iov_iter_alignment_iovec(i);  in iov_iter_alignment()
1272 	if (iov_iter_is_bvec(i))  in iov_iter_alignment()
1273 		return iov_iter_alignment_bvec(i);  in iov_iter_alignment()
1275 	if (iov_iter_is_pipe(i)) {  in iov_iter_alignment()
1276 		unsigned int p_mask = i->pipe->ring_size - 1;  in iov_iter_alignment()
1277 		size_t size = i->count;  in iov_iter_alignment()
1279 		if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))  in iov_iter_alignment()
1280 			return size | i->iov_offset;  in iov_iter_alignment()
1284 	if (iov_iter_is_xarray(i))  in iov_iter_alignment()
1285 		return (i->xarray_start + i->iov_offset) | i->count;  in iov_iter_alignment()
1291 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)  in iov_iter_gap_alignment()  argument
1295 	size_t size = i->count;  in iov_iter_gap_alignment()
1298 	if (WARN_ON(!iter_is_iovec(i)))  in iov_iter_gap_alignment()
1301 	for (k = 0; k < i->nr_segs; k++) {  in iov_iter_gap_alignment()
1302 		if (i->iov[k].iov_len) {  in iov_iter_gap_alignment()
1303 			unsigned long base = (unsigned long)i->iov[k].iov_base;  in iov_iter_gap_alignment()
1306 			v = base + i->iov[k].iov_len;  in iov_iter_gap_alignment()
1307 			if (size <= i->iov[k].iov_len)  in iov_iter_gap_alignment()
1309 			size -= i->iov[k].iov_len;  in iov_iter_gap_alignment()
1316 static inline ssize_t __pipe_get_pages(struct iov_iter *i,  in __pipe_get_pages()  argument
1322 	struct pipe_inode_info *pipe = i->pipe;  in __pipe_get_pages()
1324 	ssize_t n = push_pipe(i, maxsize, &iter_head, start);  in __pipe_get_pages()
1339 static ssize_t pipe_get_pages(struct iov_iter *i,  in pipe_get_pages()  argument
1346 	if (!sanity(i))  in pipe_get_pages()
1349 	data_start(i, &iter_head, start);  in pipe_get_pages()
1351 	npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);  in pipe_get_pages()
1354 	return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);  in pipe_get_pages()
1384 static ssize_t iter_xarray_get_pages(struct iov_iter *i,  in iter_xarray_get_pages()  argument
1396 	pos = i->xarray_start + i->iov_offset;  in iter_xarray_get_pages()
1413 	nr = iter_xarray_populate_pages(pages, i->xarray, index, count);  in iter_xarray_get_pages()
1427 static unsigned long first_iovec_segment(const struct iov_iter *i,  in first_iovec_segment()  argument
1434 	for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {  in first_iovec_segment()
1435 		unsigned long addr = (unsigned long)i->iov[k].iov_base + skip;  in first_iovec_segment()
1436 		size_t len = i->iov[k].iov_len - skip;  in first_iovec_segment()
1452 static struct page *first_bvec_segment(const struct iov_iter *i,  in first_bvec_segment()  argument
1457 	size_t skip = i->iov_offset, len;  in first_bvec_segment()
1459 	len = i->bvec->bv_len - skip;  in first_bvec_segment()
1462 	skip += i->bvec->bv_offset;  in first_bvec_segment()
1463 	page = i->bvec->bv_page + skip / PAGE_SIZE;  in first_bvec_segment()
1471 ssize_t iov_iter_get_pages(struct iov_iter *i,  in iov_iter_get_pages()  argument
1478 	if (maxsize > i->count)  in iov_iter_get_pages()
1479 		maxsize = i->count;  in iov_iter_get_pages()
1483 	if (likely(iter_is_iovec(i))) {  in iov_iter_get_pages()
1486 		addr = first_iovec_segment(i, &len, start, maxsize, maxpages);  in iov_iter_get_pages()
1489 				iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0,  in iov_iter_get_pages()
1495 	if (iov_iter_is_bvec(i)) {  in iov_iter_get_pages()
1498 		page = first_bvec_segment(i, &len, start, maxsize, maxpages);  in iov_iter_get_pages()
1504 	if (iov_iter_is_pipe(i))  in iov_iter_get_pages()
1505 		return pipe_get_pages(i, pages, maxsize, maxpages, start);  in iov_iter_get_pages()
1506 	if (iov_iter_is_xarray(i))  in iov_iter_get_pages()
1507 		return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);  in iov_iter_get_pages()
1517 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,  in pipe_get_pages_alloc()  argument
1525 	if (!sanity(i))  in pipe_get_pages_alloc()
1528 	data_start(i, &iter_head, start);  in pipe_get_pages_alloc()
1530 	npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);  in pipe_get_pages_alloc()
1539 	n = __pipe_get_pages(i, maxsize, p, iter_head, start);  in pipe_get_pages_alloc()
1547 static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i,  in iter_xarray_get_pages_alloc()  argument
1560 	pos = i->xarray_start + i->iov_offset;  in iter_xarray_get_pages_alloc()
1579 	nr = iter_xarray_populate_pages(p, i->xarray, index, count);  in iter_xarray_get_pages_alloc()
1592 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,  in iov_iter_get_pages_alloc()  argument
1600 	if (maxsize > i->count)  in iov_iter_get_pages_alloc()
1601 		maxsize = i->count;  in iov_iter_get_pages_alloc()
1605 	if (likely(iter_is_iovec(i))) {  in iov_iter_get_pages_alloc()
1608 		addr = first_iovec_segment(i, &len, start, maxsize, ~0U);  in iov_iter_get_pages_alloc()
1614 				iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0, p);  in iov_iter_get_pages_alloc()
1622 	if (iov_iter_is_bvec(i)) {  in iov_iter_get_pages_alloc()
1625 		page = first_bvec_segment(i, &len, start, maxsize, ~0U);  in iov_iter_get_pages_alloc()
1634 	if (iov_iter_is_pipe(i))  in iov_iter_get_pages_alloc()
1635 		return pipe_get_pages_alloc(i, pages, maxsize, start);  in iov_iter_get_pages_alloc()
1636 	if (iov_iter_is_xarray(i))  in iov_iter_get_pages_alloc()
1637 		return iter_xarray_get_pages_alloc(i, pages, maxsize, start);  in iov_iter_get_pages_alloc()
1643 			       struct iov_iter *i)  in csum_and_copy_from_iter()  argument
1647 	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {  in csum_and_copy_from_iter()
1651 	iterate_and_advance(i, bytes, base, len, off, ({  in csum_and_copy_from_iter()
1665 			     struct iov_iter *i)  in csum_and_copy_to_iter()  argument
1670 	if (unlikely(iov_iter_is_discard(i))) {  in csum_and_copy_to_iter()
1676 	if (unlikely(iov_iter_is_pipe(i)))  in csum_and_copy_to_iter()
1677 		bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum);  in csum_and_copy_to_iter()
1678 	else iterate_and_advance(i, bytes, base, len, off, ({  in csum_and_copy_to_iter()
1693 		struct iov_iter *i)  in hash_and_copy_to_iter()  argument
1700 	copied = copy_to_iter(addr, bytes, i);  in hash_and_copy_to_iter()
1711 static int iov_npages(const struct iov_iter *i, int maxpages)  in iov_npages()  argument
1713 	size_t skip = i->iov_offset, size = i->count;  in iov_npages()
1717 	for (p = i->iov; size; skip = 0, p++) {  in iov_npages()
1731 static int bvec_npages(const struct iov_iter *i, int maxpages)  in bvec_npages()  argument
1733 	size_t skip = i->iov_offset, size = i->count;  in bvec_npages()
1737 	for (p = i->bvec; size; skip = 0, p++) {  in bvec_npages()
1749 int iov_iter_npages(const struct iov_iter *i, int maxpages)  in iov_iter_npages()  argument
1751 	if (unlikely(!i->count))  in iov_iter_npages()
1754 	if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))  in iov_iter_npages()
1755 		return iov_npages(i, maxpages);  in iov_iter_npages()
1756 	if (iov_iter_is_bvec(i))  in iov_iter_npages()
1757 		return bvec_npages(i, maxpages);  in iov_iter_npages()
1758 	if (iov_iter_is_pipe(i)) {  in iov_iter_npages()
1763 		if (!sanity(i))  in iov_iter_npages()
1766 		data_start(i, &iter_head, &off);  in iov_iter_npages()
1768 		npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);  in iov_iter_npages()
1771 	if (iov_iter_is_xarray(i)) {  in iov_iter_npages()
1772 		unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE;  in iov_iter_npages()
1773 		int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);  in iov_iter_npages()
1806 	int ret = -EFAULT, i;  in copy_compat_iovec_from_user()  local
1811 	for (i = 0; i < nr_segs; i++) {  in copy_compat_iovec_from_user()
1815 		unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);  in copy_compat_iovec_from_user()
1816 		unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);  in copy_compat_iovec_from_user()
1823 		iov[i].iov_base = compat_ptr(buf);  in copy_compat_iovec_from_user()
1824 		iov[i].iov_len = len;  in copy_compat_iovec_from_user()
1885 		 struct iov_iter *i, bool compat)  in __import_iovec()  argument
1922 	iov_iter_init(i, type, iov, nr_segs, total_len);  in __import_iovec()
1941  * @i: Pointer to iterator that will be initialized on success.
1954 		 struct iovec **iovp, struct iov_iter *i)  in import_iovec()  argument
1956 	return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,  in import_iovec()
1962 		 struct iovec *iov, struct iov_iter *i)  in import_single_range()  argument
1971 	iov_iter_init(i, rw, iov, 1, len);  in import_single_range()
1980  * @i: &struct iov_iter to restore
1983  * Used after iov_iter_save_state() to bring restore @i, if operations may
1988 void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)  in iov_iter_restore()  argument
1990 	if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) &&  in iov_iter_restore()
1991 			 !iov_iter_is_kvec(i))  in iov_iter_restore()
1993 	i->iov_offset = state->iov_offset;  in iov_iter_restore()
1994 	i->count = state->count;  in iov_iter_restore()
2005 	if (iov_iter_is_bvec(i))  in iov_iter_restore()
2006 		i->bvec -= state->nr_segs - i->nr_segs;  in iov_iter_restore()
2008 		i->iov -= state->nr_segs - i->nr_segs;  in iov_iter_restore()
2009 	i->nr_segs = state->nr_segs;  in iov_iter_restore()