180 xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset,  in xdr_inline_pages()  argument
183 	struct kvec *head = xdr->head;  in xdr_inline_pages()
184 	struct kvec *tail = xdr->tail;  in xdr_inline_pages()
190 	xdr->pages = pages;  in xdr_inline_pages()
191 	xdr->page_base = base;  in xdr_inline_pages()
192 	xdr->page_len = len;  in xdr_inline_pages()
196 	if ((xdr->page_len & 3) == 0)  in xdr_inline_pages()
199 	xdr->buflen += len;  in xdr_inline_pages()
633 unsigned int xdr_stream_pos(const struct xdr_stream *xdr)  in xdr_stream_pos()  argument
635 	return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2;  in xdr_stream_pos()
643 unsigned int xdr_page_pos(const struct xdr_stream *xdr)  in xdr_page_pos()  argument
645 	unsigned int pos = xdr_stream_pos(xdr);  in xdr_page_pos()
647 	WARN_ON(pos < xdr->buf->head[0].iov_len);  in xdr_page_pos()
648 	return pos - xdr->buf->head[0].iov_len;  in xdr_page_pos()
666 void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,  in xdr_init_encode()  argument
672 	xdr_set_scratch_buffer(xdr, NULL, 0);  in xdr_init_encode()
674 	xdr->buf = buf;  in xdr_init_encode()
675 	xdr->iov = iov;  in xdr_init_encode()
676 	xdr->p = (__be32 *)((char *)iov->iov_base + iov->iov_len);  in xdr_init_encode()
677 	xdr->end = (__be32 *)((char *)iov->iov_base + scratch_len);  in xdr_init_encode()
680 	if (p != xdr->p && p != NULL) {  in xdr_init_encode()
683 		BUG_ON(p < xdr->p || p > xdr->end);  in xdr_init_encode()
684 		len = (char *)p - (char *)xdr->p;  in xdr_init_encode()
685 		xdr->p = p;  in xdr_init_encode()
689 	xdr->rqst = rqst;  in xdr_init_encode()
706 inline void xdr_commit_encode(struct xdr_stream *xdr)  in xdr_commit_encode()  argument
708 	int shift = xdr->scratch.iov_len;  in xdr_commit_encode()
713 	page = page_address(*xdr->page_ptr);  in xdr_commit_encode()
714 	memcpy(xdr->scratch.iov_base, page, shift);  in xdr_commit_encode()
715 	memmove(page, page + shift, (void *)xdr->p - page);  in xdr_commit_encode()
716 	xdr->scratch.iov_len = 0;  in xdr_commit_encode()
720 static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,  in xdr_get_next_encode_buffer()  argument
729 	if (xdr->buf->len + nbytes > xdr->buf->buflen)  in xdr_get_next_encode_buffer()
731 	frag1bytes = (xdr->end - xdr->p) << 2;  in xdr_get_next_encode_buffer()
733 	if (xdr->iov)  in xdr_get_next_encode_buffer()
734 		xdr->iov->iov_len += frag1bytes;  in xdr_get_next_encode_buffer()
736 		xdr->buf->page_len += frag1bytes;  in xdr_get_next_encode_buffer()
737 	xdr->page_ptr++;  in xdr_get_next_encode_buffer()
738 	xdr->iov = NULL;  in xdr_get_next_encode_buffer()
746 	xdr->scratch.iov_base = xdr->p;  in xdr_get_next_encode_buffer()
747 	xdr->scratch.iov_len = frag1bytes;  in xdr_get_next_encode_buffer()
748 	p = page_address(*xdr->page_ptr);  in xdr_get_next_encode_buffer()
753 	xdr->p = (void *)p + frag2bytes;  in xdr_get_next_encode_buffer()
754 	space_left = xdr->buf->buflen - xdr->buf->len;  in xdr_get_next_encode_buffer()
755 	xdr->end = (void *)p + min_t(int, space_left, PAGE_SIZE);  in xdr_get_next_encode_buffer()
756 	xdr->buf->page_len += frag2bytes;  in xdr_get_next_encode_buffer()
757 	xdr->buf->len += nbytes;  in xdr_get_next_encode_buffer()
760 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_get_next_encode_buffer()
773 __be32 * xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes)  in xdr_reserve_space()  argument
775 	__be32 *p = xdr->p;  in xdr_reserve_space()
778 	xdr_commit_encode(xdr);  in xdr_reserve_space()
783 	if (unlikely(q > xdr->end || q < p))  in xdr_reserve_space()
784 		return xdr_get_next_encode_buffer(xdr, nbytes);  in xdr_reserve_space()
785 	xdr->p = q;  in xdr_reserve_space()
786 	if (xdr->iov)  in xdr_reserve_space()
787 		xdr->iov->iov_len += nbytes;  in xdr_reserve_space()
789 		xdr->buf->page_len += nbytes;  in xdr_reserve_space()
790 	xdr->buf->len += nbytes;  in xdr_reserve_space()
807 int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, size_t nbytes)  in xdr_reserve_space_vec()  argument
817 	if (xdr->iov == xdr->buf->head) {  in xdr_reserve_space_vec()
818 		xdr->iov = NULL;  in xdr_reserve_space_vec()
819 		xdr->end = xdr->p;  in xdr_reserve_space_vec()
823 		thislen = xdr->buf->page_len % PAGE_SIZE;  in xdr_reserve_space_vec()
826 		p = xdr_reserve_space(xdr, thislen);  in xdr_reserve_space_vec()
859 void xdr_truncate_encode(struct xdr_stream *xdr, size_t len)  in xdr_truncate_encode()  argument
861 	struct xdr_buf *buf = xdr->buf;  in xdr_truncate_encode()
871 	xdr_commit_encode(xdr);  in xdr_truncate_encode()
877 		xdr->p = tail->iov_base + tail->iov_len;  in xdr_truncate_encode()
878 		WARN_ON_ONCE(!xdr->end);  in xdr_truncate_encode()
879 		WARN_ON_ONCE(!xdr->iov);  in xdr_truncate_encode()
889 	xdr->page_ptr = buf->pages + (new >> PAGE_SHIFT);  in xdr_truncate_encode()
892 		xdr->p = page_address(*xdr->page_ptr);  in xdr_truncate_encode()
893 		xdr->end = (void *)xdr->p + PAGE_SIZE;  in xdr_truncate_encode()
894 		xdr->p = (void *)xdr->p + (new % PAGE_SIZE);  in xdr_truncate_encode()
895 		WARN_ON_ONCE(xdr->iov);  in xdr_truncate_encode()
899 		xdr->end = head->iov_base + head->iov_len;  in xdr_truncate_encode()
901 	xdr->page_ptr--;  in xdr_truncate_encode()
904 	xdr->p = head->iov_base + head->iov_len;  in xdr_truncate_encode()
905 	xdr->iov = buf->head;  in xdr_truncate_encode()
921 int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen)  in xdr_restrict_buflen()  argument
923 	struct xdr_buf *buf = xdr->buf;  in xdr_restrict_buflen()
924 	int left_in_this_buf = (void *)xdr->end - (void *)xdr->p;  in xdr_restrict_buflen()
932 		xdr->end = (void *)xdr->end + newbuflen - end_offset;  in xdr_restrict_buflen()
946 void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, unsigned int base,  in xdr_write_pages()  argument
949 	struct xdr_buf *buf = xdr->buf;  in xdr_write_pages()
955 	iov->iov_base = (char *)xdr->p;  in xdr_write_pages()
957 	xdr->iov = iov;  in xdr_write_pages()
962 		BUG_ON(xdr->p >= xdr->end);  in xdr_write_pages()
963 		iov->iov_base = (char *)xdr->p + (len & 3);  in xdr_write_pages()
966 		*xdr->p++ = 0;  in xdr_write_pages()
973 static void xdr_set_iov(struct xdr_stream *xdr, struct kvec *iov,  in xdr_set_iov()  argument
978 	xdr->p = (__be32*)iov->iov_base;  in xdr_set_iov()
979 	xdr->end = (__be32*)(iov->iov_base + len);  in xdr_set_iov()
980 	xdr->iov = iov;  in xdr_set_iov()
981 	xdr->page_ptr = NULL;  in xdr_set_iov()
984 static int xdr_set_page_base(struct xdr_stream *xdr,  in xdr_set_page_base()  argument
993 	maxlen = xdr->buf->page_len;  in xdr_set_page_base()
1000 	base += xdr->buf->page_base;  in xdr_set_page_base()
1003 	xdr->page_ptr = &xdr->buf->pages[pgnr];  in xdr_set_page_base()
1004 	kaddr = page_address(*xdr->page_ptr);  in xdr_set_page_base()
1007 	xdr->p = (__be32*)(kaddr + pgoff);  in xdr_set_page_base()
1012 	xdr->end = (__be32*)(kaddr + pgend);  in xdr_set_page_base()
1013 	xdr->iov = NULL;  in xdr_set_page_base()
1017 static void xdr_set_page(struct xdr_stream *xdr, unsigned int base,  in xdr_set_page()  argument
1020 	if (xdr_set_page_base(xdr, base, len) < 0)  in xdr_set_page()
1021 		xdr_set_iov(xdr, xdr->buf->tail, xdr->nwords << 2);  in xdr_set_page()
1024 static void xdr_set_next_page(struct xdr_stream *xdr)  in xdr_set_next_page()  argument
1028 	newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT;  in xdr_set_next_page()
1029 	newbase -= xdr->buf->page_base;  in xdr_set_next_page()
1031 	xdr_set_page(xdr, newbase, PAGE_SIZE);  in xdr_set_next_page()
1034 static bool xdr_set_next_buffer(struct xdr_stream *xdr)  in xdr_set_next_buffer()  argument
1036 	if (xdr->page_ptr != NULL)  in xdr_set_next_buffer()
1037 		xdr_set_next_page(xdr);  in xdr_set_next_buffer()
1038 	else if (xdr->iov == xdr->buf->head) {  in xdr_set_next_buffer()
1039 		xdr_set_page(xdr, 0, PAGE_SIZE);  in xdr_set_next_buffer()
1041 	return xdr->p != xdr->end;  in xdr_set_next_buffer()
1051 void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,  in xdr_init_decode()  argument
1054 	xdr->buf = buf;  in xdr_init_decode()
1055 	xdr->scratch.iov_base = NULL;  in xdr_init_decode()
1056 	xdr->scratch.iov_len = 0;  in xdr_init_decode()
1057 	xdr->nwords = XDR_QUADLEN(buf->len);  in xdr_init_decode()
1059 		xdr_set_iov(xdr, buf->head, buf->len);  in xdr_init_decode()
1061 		xdr_set_page_base(xdr, 0, buf->len);  in xdr_init_decode()
1063 		xdr_set_iov(xdr, buf->head, buf->len);  in xdr_init_decode()
1064 	if (p != NULL && p > xdr->p && xdr->end >= p) {  in xdr_init_decode()
1065 		xdr->nwords -= p - xdr->p;  in xdr_init_decode()
1066 		xdr->p = p;  in xdr_init_decode()
1068 	xdr->rqst = rqst;  in xdr_init_decode()
1079 void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,  in xdr_init_decode_pages()  argument
1087 	xdr_init_decode(xdr, buf, NULL, NULL);  in xdr_init_decode_pages()
1091 static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)  in __xdr_inline_decode()  argument
1094 	__be32 *p = xdr->p;  in __xdr_inline_decode()
1097 	if (unlikely(nwords > xdr->nwords || q > xdr->end || q < p))  in __xdr_inline_decode()
1099 	xdr->p = q;  in __xdr_inline_decode()
1100 	xdr->nwords -= nwords;  in __xdr_inline_decode()
1115 void xdr_set_scratch_buffer(struct xdr_stream *xdr, void *buf, size_t buflen)  in xdr_set_scratch_buffer()  argument
1117 	xdr->scratch.iov_base = buf;  in xdr_set_scratch_buffer()
1118 	xdr->scratch.iov_len = buflen;  in xdr_set_scratch_buffer()
1122 static __be32 *xdr_copy_to_scratch(struct xdr_stream *xdr, size_t nbytes)  in xdr_copy_to_scratch()  argument
1125 	char *cpdest = xdr->scratch.iov_base;  in xdr_copy_to_scratch()
1126 	size_t cplen = (char *)xdr->end - (char *)xdr->p;  in xdr_copy_to_scratch()
1128 	if (nbytes > xdr->scratch.iov_len)  in xdr_copy_to_scratch()
1130 	p = __xdr_inline_decode(xdr, cplen);  in xdr_copy_to_scratch()
1134 	if (!xdr_set_next_buffer(xdr))  in xdr_copy_to_scratch()
1138 	p = __xdr_inline_decode(xdr, nbytes);  in xdr_copy_to_scratch()
1142 	return xdr->scratch.iov_base;  in xdr_copy_to_scratch()
1144 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_copy_to_scratch()
1158 __be32 * xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)  in xdr_inline_decode()  argument
1163 		return xdr->p;  in xdr_inline_decode()
1164 	if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))  in xdr_inline_decode()
1166 	p = __xdr_inline_decode(xdr, nbytes);  in xdr_inline_decode()
1169 	return xdr_copy_to_scratch(xdr, nbytes);  in xdr_inline_decode()
1171 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_inline_decode()
1176 static void xdr_realign_pages(struct xdr_stream *xdr)  in xdr_realign_pages()  argument
1178 	struct xdr_buf *buf = xdr->buf;  in xdr_realign_pages()
1180 	unsigned int cur = xdr_stream_pos(xdr);  in xdr_realign_pages()
1187 		trace_rpc_xdr_alignment(xdr, offset, copied);  in xdr_realign_pages()
1188 		xdr->nwords = XDR_QUADLEN(buf->len - cur);  in xdr_realign_pages()
1192 static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_align_pages()  argument
1194 	struct xdr_buf *buf = xdr->buf;  in xdr_align_pages()
1196 	unsigned int cur = xdr_stream_pos(xdr);  in xdr_align_pages()
1199 	if (xdr->nwords == 0)  in xdr_align_pages()
1202 	xdr_realign_pages(xdr);  in xdr_align_pages()
1203 	if (nwords > xdr->nwords) {  in xdr_align_pages()
1204 		nwords = xdr->nwords;  in xdr_align_pages()
1209 	else if (nwords < xdr->nwords) {  in xdr_align_pages()
1213 		trace_rpc_xdr_alignment(xdr, offset, copied);  in xdr_align_pages()
1214 		xdr->nwords = XDR_QUADLEN(buf->len - cur);  in xdr_align_pages()
1230 unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_read_pages()  argument
1232 	struct xdr_buf *buf = xdr->buf;  in xdr_read_pages()
1238 	len = xdr_align_pages(xdr, len);  in xdr_read_pages()
1243 	xdr->iov = iov = buf->tail;  in xdr_read_pages()
1245 	end = ((xdr->nwords - nwords) << 2) + padding;  in xdr_read_pages()
1253 	xdr->p = (__be32 *)((char *)iov->iov_base + padding);  in xdr_read_pages()
1254 	xdr->end = (__be32 *)((char *)iov->iov_base + end);  in xdr_read_pages()
1255 	xdr->page_ptr = NULL;  in xdr_read_pages()
1256 	xdr->nwords = XDR_QUADLEN(end - padding);  in xdr_read_pages()
1261 uint64_t xdr_align_data(struct xdr_stream *xdr, uint64_t offset, uint32_t length)  in xdr_align_data()  argument
1263 	struct xdr_buf *buf = xdr->buf;  in xdr_align_data()
1271 	xdr_realign_pages(xdr);  in xdr_align_data()
1272 	from = xdr_page_pos(xdr);  in xdr_align_data()
1273 	bytes = xdr->nwords << 2;  in xdr_align_data()
1291 	xdr->nwords -= XDR_QUADLEN(length);  in xdr_align_data()
1292 	xdr_set_page(xdr, from + length, PAGE_SIZE);  in xdr_align_data()
1297 uint64_t xdr_expand_hole(struct xdr_stream *xdr, uint64_t offset, uint64_t length)  in xdr_expand_hole()  argument
1299 	struct xdr_buf *buf = xdr->buf;  in xdr_expand_hole()
1308 	xdr_realign_pages(xdr);  in xdr_expand_hole()
1309 	from = xdr_page_pos(xdr);  in xdr_expand_hole()
1310 	bytes = xdr->nwords << 2;  in xdr_expand_hole()
1316 		xdr->nwords -= XDR_QUADLEN(truncated);  in xdr_expand_hole()
1329 	xdr_set_page(xdr, offset + length, PAGE_SIZE);  in xdr_expand_hole()
1344 void xdr_enter_page(struct xdr_stream *xdr, unsigned int len)  in xdr_enter_page()  argument
1346 	len = xdr_align_pages(xdr, len);  in xdr_enter_page()
1352 		xdr_set_page_base(xdr, 0, len);  in xdr_enter_page()
1853 ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size)  in xdr_stream_decode_opaque()  argument
1858 	ret = xdr_stream_decode_opaque_inline(xdr, &p, size);  in xdr_stream_decode_opaque()
1879 ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr,  in xdr_stream_decode_opaque_dup()  argument
1885 	ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);  in xdr_stream_decode_opaque_dup()
1908 ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size)  in xdr_stream_decode_string()  argument
1913 	ret = xdr_stream_decode_opaque_inline(xdr, &p, size);  in xdr_stream_decode_string()
1937 ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,  in xdr_stream_decode_string_dup()  argument
1943 	ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);  in xdr_stream_decode_string_dup()