Lines Matching full:i
18 #define iterate_buf(i, n, base, len, off, __p, STEP) { \ argument
21 base = __p + i->iov_offset; \
23 i->iov_offset += len; \
28 #define iterate_iovec(i, n, base, len, off, __p, STEP) { \ argument
30 size_t skip = i->iov_offset; \
45 i->iov_offset = skip; \
49 #define iterate_bvec(i, n, base, len, off, p, STEP) { \ argument
51 unsigned skip = i->iov_offset; \
73 i->iov_offset = skip; \
77 #define iterate_xarray(i, n, base, len, __off, STEP) { \ argument
81 loff_t start = i->xarray_start + i->iov_offset; \
83 XA_STATE(xas, i->xarray, index); \
113 i->iov_offset += __off; \
117 #define __iterate_and_advance(i, n, base, len, off, I, K) { \ argument
118 if (unlikely(i->count < n)) \
119 n = i->count; \
121 if (likely(iter_is_ubuf(i))) { \
124 iterate_buf(i, n, base, len, off, \
125 i->ubuf, (I)) \
126 } else if (likely(iter_is_iovec(i))) { \
127 const struct iovec *iov = iter_iov(i); \
130 iterate_iovec(i, n, base, len, off, \
131 iov, (I)) \
132 i->nr_segs -= iov - iter_iov(i); \
133 i->__iov = iov; \
134 } else if (iov_iter_is_bvec(i)) { \
135 const struct bio_vec *bvec = i->bvec; \
138 iterate_bvec(i, n, base, len, off, \
140 i->nr_segs -= bvec - i->bvec; \
141 i->bvec = bvec; \
142 } else if (iov_iter_is_kvec(i)) { \
143 const struct kvec *kvec = i->kvec; \
146 iterate_iovec(i, n, base, len, off, \
148 i->nr_segs -= kvec - i->kvec; \
149 i->kvec = kvec; \
150 } else if (iov_iter_is_xarray(i)) { \
153 iterate_xarray(i, n, base, len, off, \
156 i->count -= n; \
159 #define iterate_and_advance(i, n, base, len, off, I, K) \ argument
160 __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0))
201 * @i: iterator
212 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) in fault_in_iov_iter_readable() argument
214 if (iter_is_ubuf(i)) { in fault_in_iov_iter_readable()
215 size_t n = min(size, iov_iter_count(i)); in fault_in_iov_iter_readable()
216 n -= fault_in_readable(i->ubuf + i->iov_offset, n); in fault_in_iov_iter_readable()
218 } else if (iter_is_iovec(i)) { in fault_in_iov_iter_readable()
219 size_t count = min(size, iov_iter_count(i)); in fault_in_iov_iter_readable()
224 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) { in fault_in_iov_iter_readable()
243 * @i: iterator
246 * Faults in the iterator using get_user_pages(), i.e., without triggering
248 * some or all of the pages in @i aren't in memory.
255 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) in fault_in_iov_iter_writeable() argument
257 if (iter_is_ubuf(i)) { in fault_in_iov_iter_writeable()
258 size_t n = min(size, iov_iter_count(i)); in fault_in_iov_iter_writeable()
259 n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n); in fault_in_iov_iter_writeable()
261 } else if (iter_is_iovec(i)) { in fault_in_iov_iter_writeable()
262 size_t count = min(size, iov_iter_count(i)); in fault_in_iov_iter_writeable()
267 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) { in fault_in_iov_iter_writeable()
284 void iov_iter_init(struct iov_iter *i, unsigned int direction, in iov_iter_init() argument
289 *i = (struct iov_iter) { in iov_iter_init()
310 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_to_iter() argument
312 if (WARN_ON_ONCE(i->data_source)) in _copy_to_iter()
314 if (user_backed_iter(i)) in _copy_to_iter()
316 iterate_and_advance(i, bytes, base, len, off, in _copy_to_iter()
339 * @i: destination iterator
359 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_mc_to_iter() argument
361 if (WARN_ON_ONCE(i->data_source)) in _copy_mc_to_iter()
363 if (user_backed_iter(i)) in _copy_mc_to_iter()
365 __iterate_and_advance(i, bytes, base, len, off, in _copy_mc_to_iter()
375 static void *memcpy_from_iter(struct iov_iter *i, void *to, const void *from, in memcpy_from_iter() argument
378 if (iov_iter_is_copy_mc(i)) in memcpy_from_iter()
383 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter() argument
385 if (WARN_ON_ONCE(!i->data_source)) in _copy_from_iter()
388 if (user_backed_iter(i)) in _copy_from_iter()
390 iterate_and_advance(i, bytes, base, len, off, in _copy_from_iter()
392 memcpy_from_iter(i, addr + off, base, len) in _copy_from_iter()
399 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_nocache() argument
401 if (WARN_ON_ONCE(!i->data_source)) in _copy_from_iter_nocache()
404 iterate_and_advance(i, bytes, base, len, off, in _copy_from_iter_nocache()
418 * @i: source iterator
430 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_flushcache() argument
432 if (WARN_ON_ONCE(!i->data_source)) in _copy_from_iter_flushcache()
435 iterate_and_advance(i, bytes, base, len, off, in _copy_from_iter_flushcache()
469 struct iov_iter *i) in copy_page_to_iter() argument
474 if (WARN_ON_ONCE(i->data_source)) in copy_page_to_iter()
481 n = _copy_to_iter(kaddr + offset, n, i); in copy_page_to_iter()
498 struct iov_iter *i) in copy_page_to_iter_nofault() argument
504 if (WARN_ON_ONCE(i->data_source)) in copy_page_to_iter_nofault()
512 iterate_and_advance(i, n, base, len, off, in copy_page_to_iter_nofault()
532 struct iov_iter *i) in copy_page_from_iter() argument
542 n = _copy_from_iter(kaddr + offset, n, i); in copy_page_from_iter()
558 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) in iov_iter_zero() argument
560 iterate_and_advance(i, bytes, base, len, count, in iov_iter_zero()
570 size_t bytes, struct iov_iter *i) in copy_page_from_iter_atomic() argument
576 if (WARN_ON_ONCE(!i->data_source)) in copy_page_from_iter_atomic()
590 iterate_and_advance(i, n, base, len, off, in copy_page_from_iter_atomic()
592 memcpy_from_iter(i, p + off, base, len) in copy_page_from_iter_atomic()
603 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) in iov_iter_bvec_advance() argument
607 if (!i->count) in iov_iter_bvec_advance()
609 i->count -= size; in iov_iter_bvec_advance()
611 size += i->iov_offset; in iov_iter_bvec_advance()
613 for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) { in iov_iter_bvec_advance()
618 i->iov_offset = size; in iov_iter_bvec_advance()
619 i->nr_segs -= bvec - i->bvec; in iov_iter_bvec_advance()
620 i->bvec = bvec; in iov_iter_bvec_advance()
623 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) in iov_iter_iovec_advance() argument
627 if (!i->count) in iov_iter_iovec_advance()
629 i->count -= size; in iov_iter_iovec_advance()
631 size += i->iov_offset; // from beginning of current segment in iov_iter_iovec_advance()
632 for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) { in iov_iter_iovec_advance()
637 i->iov_offset = size; in iov_iter_iovec_advance()
638 i->nr_segs -= iov - iter_iov(i); in iov_iter_iovec_advance()
639 i->__iov = iov; in iov_iter_iovec_advance()
642 void iov_iter_advance(struct iov_iter *i, size_t size) in iov_iter_advance() argument
644 if (unlikely(i->count < size)) in iov_iter_advance()
645 size = i->count; in iov_iter_advance()
646 if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) { in iov_iter_advance()
647 i->iov_offset += size; in iov_iter_advance()
648 i->count -= size; in iov_iter_advance()
649 } else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { in iov_iter_advance()
651 iov_iter_iovec_advance(i, size); in iov_iter_advance()
652 } else if (iov_iter_is_bvec(i)) { in iov_iter_advance()
653 iov_iter_bvec_advance(i, size); in iov_iter_advance()
654 } else if (iov_iter_is_discard(i)) { in iov_iter_advance()
655 i->count -= size; in iov_iter_advance()
660 void iov_iter_revert(struct iov_iter *i, size_t unroll) in iov_iter_revert() argument
666 i->count += unroll; in iov_iter_revert()
667 if (unlikely(iov_iter_is_discard(i))) in iov_iter_revert()
669 if (unroll <= i->iov_offset) { in iov_iter_revert()
670 i->iov_offset -= unroll; in iov_iter_revert()
673 unroll -= i->iov_offset; in iov_iter_revert()
674 if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) { in iov_iter_revert()
679 } else if (iov_iter_is_bvec(i)) { in iov_iter_revert()
680 const struct bio_vec *bvec = i->bvec; in iov_iter_revert()
683 i->nr_segs++; in iov_iter_revert()
685 i->bvec = bvec; in iov_iter_revert()
686 i->iov_offset = n - unroll; in iov_iter_revert()
692 const struct iovec *iov = iter_iov(i); in iov_iter_revert()
695 i->nr_segs++; in iov_iter_revert()
697 i->__iov = iov; in iov_iter_revert()
698 i->iov_offset = n - unroll; in iov_iter_revert()
710 size_t iov_iter_single_seg_count(const struct iov_iter *i) in iov_iter_single_seg_count() argument
712 if (i->nr_segs > 1) { in iov_iter_single_seg_count()
713 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_single_seg_count()
714 return min(i->count, iter_iov(i)->iov_len - i->iov_offset); in iov_iter_single_seg_count()
715 if (iov_iter_is_bvec(i)) in iov_iter_single_seg_count()
716 return min(i->count, i->bvec->bv_len - i->iov_offset); in iov_iter_single_seg_count()
718 return i->count; in iov_iter_single_seg_count()
722 void iov_iter_kvec(struct iov_iter *i, unsigned int direction, in iov_iter_kvec() argument
727 *i = (struct iov_iter){ in iov_iter_kvec()
739 void iov_iter_bvec(struct iov_iter *i, unsigned int direction, in iov_iter_bvec() argument
744 *i = (struct iov_iter){ in iov_iter_bvec()
757 * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
758 * @i: The iterator to initialise.
762 * @count: The size of the I/O buffer in bytes.
764 * Set up an I/O iterator to either draw data out of the pages attached to an
769 void iov_iter_xarray(struct iov_iter *i, unsigned int direction, in iov_iter_xarray() argument
773 *i = (struct iov_iter) { in iov_iter_xarray()
786 * iov_iter_discard - Initialise an I/O iterator that discards data
787 * @i: The iterator to initialise.
789 * @count: The size of the I/O buffer in bytes.
791 * Set up an I/O iterator that just discards everything that's written to it.
794 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) in iov_iter_discard() argument
797 *i = (struct iov_iter){ in iov_iter_discard()
807 static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask, in iov_iter_aligned_iovec() argument
810 size_t size = i->count; in iov_iter_aligned_iovec()
811 size_t skip = i->iov_offset; in iov_iter_aligned_iovec()
814 for (k = 0; k < i->nr_segs; k++, skip = 0) { in iov_iter_aligned_iovec()
815 const struct iovec *iov = iter_iov(i) + k; in iov_iter_aligned_iovec()
832 static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask, in iov_iter_aligned_bvec() argument
835 size_t size = i->count; in iov_iter_aligned_bvec()
836 unsigned skip = i->iov_offset; in iov_iter_aligned_bvec()
839 for (k = 0; k < i->nr_segs; k++, skip = 0) { in iov_iter_aligned_bvec()
840 size_t len = i->bvec[k].bv_len - skip; in iov_iter_aligned_bvec()
846 if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask) in iov_iter_aligned_bvec()
860 * @i: &struct iov_iter to restore
866 bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask, in iov_iter_is_aligned() argument
869 if (likely(iter_is_ubuf(i))) { in iov_iter_is_aligned()
870 if (i->count & len_mask) in iov_iter_is_aligned()
872 if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask) in iov_iter_is_aligned()
877 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_is_aligned()
878 return iov_iter_aligned_iovec(i, addr_mask, len_mask); in iov_iter_is_aligned()
880 if (iov_iter_is_bvec(i)) in iov_iter_is_aligned()
881 return iov_iter_aligned_bvec(i, addr_mask, len_mask); in iov_iter_is_aligned()
883 if (iov_iter_is_xarray(i)) { in iov_iter_is_aligned()
884 if (i->count & len_mask) in iov_iter_is_aligned()
886 if ((i->xarray_start + i->iov_offset) & addr_mask) in iov_iter_is_aligned()
894 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) in iov_iter_alignment_iovec() argument
897 size_t size = i->count; in iov_iter_alignment_iovec()
898 size_t skip = i->iov_offset; in iov_iter_alignment_iovec()
901 for (k = 0; k < i->nr_segs; k++, skip = 0) { in iov_iter_alignment_iovec()
902 const struct iovec *iov = iter_iov(i) + k; in iov_iter_alignment_iovec()
917 static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) in iov_iter_alignment_bvec() argument
920 size_t size = i->count; in iov_iter_alignment_bvec()
921 unsigned skip = i->iov_offset; in iov_iter_alignment_bvec()
924 for (k = 0; k < i->nr_segs; k++, skip = 0) { in iov_iter_alignment_bvec()
925 size_t len = i->bvec[k].bv_len - skip; in iov_iter_alignment_bvec()
926 res |= (unsigned long)i->bvec[k].bv_offset + skip; in iov_iter_alignment_bvec()
937 unsigned long iov_iter_alignment(const struct iov_iter *i) in iov_iter_alignment() argument
939 if (likely(iter_is_ubuf(i))) { in iov_iter_alignment()
940 size_t size = i->count; in iov_iter_alignment()
942 return ((unsigned long)i->ubuf + i->iov_offset) | size; in iov_iter_alignment()
947 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_alignment()
948 return iov_iter_alignment_iovec(i); in iov_iter_alignment()
950 if (iov_iter_is_bvec(i)) in iov_iter_alignment()
951 return iov_iter_alignment_bvec(i); in iov_iter_alignment()
953 if (iov_iter_is_xarray(i)) in iov_iter_alignment()
954 return (i->xarray_start + i->iov_offset) | i->count; in iov_iter_alignment()
960 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) in iov_iter_gap_alignment() argument
964 size_t size = i->count; in iov_iter_gap_alignment()
967 if (iter_is_ubuf(i)) in iov_iter_gap_alignment()
970 if (WARN_ON(!iter_is_iovec(i))) in iov_iter_gap_alignment()
973 for (k = 0; k < i->nr_segs; k++) { in iov_iter_gap_alignment()
974 const struct iovec *iov = iter_iov(i) + k; in iov_iter_gap_alignment()
1032 static ssize_t iter_xarray_get_pages(struct iov_iter *i, in iter_xarray_get_pages() argument
1040 pos = i->xarray_start + i->iov_offset; in iter_xarray_get_pages()
1048 nr = iter_xarray_populate_pages(*pages, i->xarray, index, count); in iter_xarray_get_pages()
1053 i->iov_offset += maxsize; in iter_xarray_get_pages()
1054 i->count -= maxsize; in iter_xarray_get_pages()
1059 static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size) in first_iovec_segment() argument
1064 if (iter_is_ubuf(i)) in first_iovec_segment()
1065 return (unsigned long)i->ubuf + i->iov_offset; in first_iovec_segment()
1067 for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { in first_iovec_segment()
1068 const struct iovec *iov = iter_iov(i) + k; in first_iovec_segment()
1081 static struct page *first_bvec_segment(const struct iov_iter *i, in first_bvec_segment() argument
1085 size_t skip = i->iov_offset, len; in first_bvec_segment()
1087 len = i->bvec->bv_len - skip; in first_bvec_segment()
1090 skip += i->bvec->bv_offset; in first_bvec_segment()
1091 page = i->bvec->bv_page + skip / PAGE_SIZE; in first_bvec_segment()
1096 static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i, in __iov_iter_get_pages_alloc() argument
1102 if (maxsize > i->count) in __iov_iter_get_pages_alloc()
1103 maxsize = i->count; in __iov_iter_get_pages_alloc()
1109 if (likely(user_backed_iter(i))) { in __iov_iter_get_pages_alloc()
1113 if (iov_iter_rw(i) != WRITE) in __iov_iter_get_pages_alloc()
1115 if (i->nofault) in __iov_iter_get_pages_alloc()
1118 addr = first_iovec_segment(i, &maxsize); in __iov_iter_get_pages_alloc()
1128 iov_iter_advance(i, maxsize); in __iov_iter_get_pages_alloc()
1131 if (iov_iter_is_bvec(i)) { in __iov_iter_get_pages_alloc()
1135 page = first_bvec_segment(i, &maxsize, start); in __iov_iter_get_pages_alloc()
1143 i->count -= maxsize; in __iov_iter_get_pages_alloc()
1144 i->iov_offset += maxsize; in __iov_iter_get_pages_alloc()
1145 if (i->iov_offset == i->bvec->bv_len) { in __iov_iter_get_pages_alloc()
1146 i->iov_offset = 0; in __iov_iter_get_pages_alloc()
1147 i->bvec++; in __iov_iter_get_pages_alloc()
1148 i->nr_segs--; in __iov_iter_get_pages_alloc()
1152 if (iov_iter_is_xarray(i)) in __iov_iter_get_pages_alloc()
1153 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); in __iov_iter_get_pages_alloc()
1157 ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages, in iov_iter_get_pages2() argument
1164 return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start); in iov_iter_get_pages2()
1168 ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, in iov_iter_get_pages_alloc2() argument
1175 len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start); in iov_iter_get_pages_alloc2()
1185 struct iov_iter *i) in csum_and_copy_from_iter() argument
1189 if (WARN_ON_ONCE(!i->data_source)) in csum_and_copy_from_iter()
1192 iterate_and_advance(i, bytes, base, len, off, ({ in csum_and_copy_from_iter()
1206 struct iov_iter *i) in csum_and_copy_to_iter() argument
1211 if (WARN_ON_ONCE(i->data_source)) in csum_and_copy_to_iter()
1213 if (unlikely(iov_iter_is_discard(i))) { in csum_and_copy_to_iter()
1223 iterate_and_advance(i, bytes, base, len, off, ({ in csum_and_copy_to_iter()
1238 struct iov_iter *i) in hash_and_copy_to_iter() argument
1245 copied = copy_to_iter(addr, bytes, i); in hash_and_copy_to_iter()
1256 static int iov_npages(const struct iov_iter *i, int maxpages) in iov_npages() argument
1258 size_t skip = i->iov_offset, size = i->count; in iov_npages()
1262 for (p = iter_iov(i); size; skip = 0, p++) { in iov_npages()
1276 static int bvec_npages(const struct iov_iter *i, int maxpages) in bvec_npages() argument
1278 size_t skip = i->iov_offset, size = i->count; in bvec_npages()
1282 for (p = i->bvec; size; skip = 0, p++) { in bvec_npages()
1294 int iov_iter_npages(const struct iov_iter *i, int maxpages) in iov_iter_npages() argument
1296 if (unlikely(!i->count)) in iov_iter_npages()
1298 if (likely(iter_is_ubuf(i))) { in iov_iter_npages()
1299 unsigned offs = offset_in_page(i->ubuf + i->iov_offset); in iov_iter_npages()
1300 int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE); in iov_iter_npages()
1304 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_npages()
1305 return iov_npages(i, maxpages); in iov_iter_npages()
1306 if (iov_iter_is_bvec(i)) in iov_iter_npages()
1307 return bvec_npages(i, maxpages); in iov_iter_npages()
1308 if (iov_iter_is_xarray(i)) { in iov_iter_npages()
1309 unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; in iov_iter_npages()
1310 int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); in iov_iter_npages()
1338 int ret = -EFAULT, i; in copy_compat_iovec_from_user() local
1343 for (i = 0; i < nr_segs; i++) { in copy_compat_iovec_from_user()
1347 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end); in copy_compat_iovec_from_user()
1348 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end); in copy_compat_iovec_from_user()
1355 iov[i].iov_base = compat_ptr(buf); in copy_compat_iovec_from_user()
1356 iov[i].iov_len = len; in copy_compat_iovec_from_user()
1436 struct iovec **iovp, struct iov_iter *i, in __import_iovec_ubuf() argument
1449 ret = import_ubuf(type, iov->iov_base, iov->iov_len, i); in __import_iovec_ubuf()
1453 return i->count; in __import_iovec_ubuf()
1458 struct iov_iter *i, bool compat) in __import_iovec() argument
1465 return __import_iovec_ubuf(type, uvec, iovp, i, compat); in __import_iovec()
1498 iov_iter_init(i, type, iov, nr_segs, total_len); in __import_iovec()
1517 * @i: Pointer to iterator that will be initialized on success.
1530 struct iovec **iovp, struct iov_iter *i) in import_iovec() argument
1532 return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, in import_iovec()
1538 struct iovec *iov, struct iov_iter *i) in import_single_range() argument
1545 iov_iter_ubuf(i, rw, buf, len); in import_single_range()
1550 int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) in import_ubuf() argument
1557 iov_iter_ubuf(i, rw, buf, len); in import_ubuf()
1566 * @i: &struct iov_iter to restore
1569 * Used after iov_iter_save_state() to bring restore @i, if operations may
1574 void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) in iov_iter_restore() argument
1576 if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) && in iov_iter_restore()
1577 !iter_is_ubuf(i)) && !iov_iter_is_kvec(i)) in iov_iter_restore()
1579 i->iov_offset = state->iov_offset; in iov_iter_restore()
1580 i->count = state->count; in iov_iter_restore()
1581 if (iter_is_ubuf(i)) in iov_iter_restore()
1593 if (iov_iter_is_bvec(i)) in iov_iter_restore()
1594 i->bvec -= state->nr_segs - i->nr_segs; in iov_iter_restore()
1596 i->__iov -= state->nr_segs - i->nr_segs; in iov_iter_restore()
1597 i->nr_segs = state->nr_segs; in iov_iter_restore()
1604 static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, in iov_iter_extract_xarray_pages() argument
1612 loff_t pos = i->xarray_start + i->iov_offset; in iov_iter_extract_xarray_pages()
1614 XA_STATE(xas, i->xarray, index); in iov_iter_extract_xarray_pages()
1642 iov_iter_advance(i, maxsize); in iov_iter_extract_xarray_pages()
1650 static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, in iov_iter_extract_bvec_pages() argument
1657 size_t skip = i->iov_offset, offset, size; in iov_iter_extract_bvec_pages()
1661 if (i->nr_segs == 0) in iov_iter_extract_bvec_pages()
1663 size = min(maxsize, i->bvec->bv_len - skip); in iov_iter_extract_bvec_pages()
1666 i->iov_offset = 0; in iov_iter_extract_bvec_pages()
1667 i->nr_segs--; in iov_iter_extract_bvec_pages()
1668 i->bvec++; in iov_iter_extract_bvec_pages()
1672 skip += i->bvec->bv_offset; in iov_iter_extract_bvec_pages()
1673 page = i->bvec->bv_page + skip / PAGE_SIZE; in iov_iter_extract_bvec_pages()
1685 iov_iter_advance(i, size); in iov_iter_extract_bvec_pages()
1693 static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i, in iov_iter_extract_kvec_pages() argument
1701 size_t skip = i->iov_offset, offset, len, size; in iov_iter_extract_kvec_pages()
1705 if (i->nr_segs == 0) in iov_iter_extract_kvec_pages()
1707 size = min(maxsize, i->kvec->iov_len - skip); in iov_iter_extract_kvec_pages()
1710 i->iov_offset = 0; in iov_iter_extract_kvec_pages()
1711 i->nr_segs--; in iov_iter_extract_kvec_pages()
1712 i->kvec++; in iov_iter_extract_kvec_pages()
1716 kaddr = i->kvec->iov_base + skip; in iov_iter_extract_kvec_pages()
1741 iov_iter_advance(i, size); in iov_iter_extract_kvec_pages()
1757 static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, in iov_iter_extract_user_pages() argument
1769 if (i->data_source == ITER_DEST) in iov_iter_extract_user_pages()
1773 if (i->nofault) in iov_iter_extract_user_pages()
1776 addr = first_iovec_segment(i, &maxsize); in iov_iter_extract_user_pages()
1786 iov_iter_advance(i, maxsize); in iov_iter_extract_user_pages()
1792 * @i: The iterator to extract from
1833 ssize_t iov_iter_extract_pages(struct iov_iter *i, in iov_iter_extract_pages() argument
1840 maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT); in iov_iter_extract_pages()
1844 if (likely(user_backed_iter(i))) in iov_iter_extract_pages()
1845 return iov_iter_extract_user_pages(i, pages, maxsize, in iov_iter_extract_pages()
1848 if (iov_iter_is_kvec(i)) in iov_iter_extract_pages()
1849 return iov_iter_extract_kvec_pages(i, pages, maxsize, in iov_iter_extract_pages()
1852 if (iov_iter_is_bvec(i)) in iov_iter_extract_pages()
1853 return iov_iter_extract_bvec_pages(i, pages, maxsize, in iov_iter_extract_pages()
1856 if (iov_iter_is_xarray(i)) in iov_iter_extract_pages()
1857 return iov_iter_extract_xarray_pages(i, pages, maxsize, in iov_iter_extract_pages()