178 xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset,  in xdr_inline_pages()  argument
181 	struct kvec *head = xdr->head;  in xdr_inline_pages()
182 	struct kvec *tail = xdr->tail;  in xdr_inline_pages()
188 	xdr->pages = pages;  in xdr_inline_pages()
189 	xdr->page_base = base;  in xdr_inline_pages()
190 	xdr->page_len = len;  in xdr_inline_pages()
194 	xdr->buflen += len;  in xdr_inline_pages()
849 unsigned int xdr_stream_pos(const struct xdr_stream *xdr)  in xdr_stream_pos()  argument
851 	return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2;  in xdr_stream_pos()
855 static void xdr_stream_set_pos(struct xdr_stream *xdr, unsigned int pos)  in xdr_stream_set_pos()  argument
857 	unsigned int blen = xdr->buf->len;  in xdr_stream_set_pos()
859 	xdr->nwords = blen > pos ? XDR_QUADLEN(blen) - XDR_QUADLEN(pos) : 0;  in xdr_stream_set_pos()
862 static void xdr_stream_page_set_pos(struct xdr_stream *xdr, unsigned int pos)  in xdr_stream_page_set_pos()  argument
864 	xdr_stream_set_pos(xdr, pos + xdr->buf->head[0].iov_len);  in xdr_stream_page_set_pos()
871 unsigned int xdr_page_pos(const struct xdr_stream *xdr)  in xdr_page_pos()  argument
873 	unsigned int pos = xdr_stream_pos(xdr);  in xdr_page_pos()
875 	WARN_ON(pos < xdr->buf->head[0].iov_len);  in xdr_page_pos()
876 	return pos - xdr->buf->head[0].iov_len;  in xdr_page_pos()
894 void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,  in xdr_init_encode()  argument
900 	xdr_reset_scratch_buffer(xdr);  in xdr_init_encode()
902 	xdr->buf = buf;  in xdr_init_encode()
903 	xdr->iov = iov;  in xdr_init_encode()
904 	xdr->p = (__be32 *)((char *)iov->iov_base + iov->iov_len);  in xdr_init_encode()
905 	xdr->end = (__be32 *)((char *)iov->iov_base + scratch_len);  in xdr_init_encode()
908 	if (p != xdr->p && p != NULL) {  in xdr_init_encode()
911 		BUG_ON(p < xdr->p || p > xdr->end);  in xdr_init_encode()
912 		len = (char *)p - (char *)xdr->p;  in xdr_init_encode()
913 		xdr->p = p;  in xdr_init_encode()
917 	xdr->rqst = rqst;  in xdr_init_encode()
934 inline void xdr_commit_encode(struct xdr_stream *xdr)  in xdr_commit_encode()  argument
936 	int shift = xdr->scratch.iov_len;  in xdr_commit_encode()
941 	page = page_address(*xdr->page_ptr);  in xdr_commit_encode()
942 	memcpy(xdr->scratch.iov_base, page, shift);  in xdr_commit_encode()
943 	memmove(page, page + shift, (void *)xdr->p - page);  in xdr_commit_encode()
944 	xdr_reset_scratch_buffer(xdr);  in xdr_commit_encode()
948 static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,  in xdr_get_next_encode_buffer()  argument
957 	if (xdr->buf->len + nbytes > xdr->buf->buflen)  in xdr_get_next_encode_buffer()
959 	frag1bytes = (xdr->end - xdr->p) << 2;  in xdr_get_next_encode_buffer()
961 	if (xdr->iov)  in xdr_get_next_encode_buffer()
962 		xdr->iov->iov_len += frag1bytes;  in xdr_get_next_encode_buffer()
964 		xdr->buf->page_len += frag1bytes;  in xdr_get_next_encode_buffer()
965 	xdr->page_ptr++;  in xdr_get_next_encode_buffer()
966 	xdr->iov = NULL;  in xdr_get_next_encode_buffer()
974 	xdr_set_scratch_buffer(xdr, xdr->p, frag1bytes);  in xdr_get_next_encode_buffer()
975 	p = page_address(*xdr->page_ptr);  in xdr_get_next_encode_buffer()
980 	xdr->p = (void *)p + frag2bytes;  in xdr_get_next_encode_buffer()
981 	space_left = xdr->buf->buflen - xdr->buf->len;  in xdr_get_next_encode_buffer()
982 	xdr->end = (void *)p + min_t(int, space_left, PAGE_SIZE);  in xdr_get_next_encode_buffer()
983 	xdr->buf->page_len += frag2bytes;  in xdr_get_next_encode_buffer()
984 	xdr->buf->len += nbytes;  in xdr_get_next_encode_buffer()
987 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_get_next_encode_buffer()
1000 __be32 * xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes)  in xdr_reserve_space()  argument
1002 	__be32 *p = xdr->p;  in xdr_reserve_space()
1005 	xdr_commit_encode(xdr);  in xdr_reserve_space()
1010 	if (unlikely(q > xdr->end || q < p))  in xdr_reserve_space()
1011 		return xdr_get_next_encode_buffer(xdr, nbytes);  in xdr_reserve_space()
1012 	xdr->p = q;  in xdr_reserve_space()
1013 	if (xdr->iov)  in xdr_reserve_space()
1014 		xdr->iov->iov_len += nbytes;  in xdr_reserve_space()
1016 		xdr->buf->page_len += nbytes;  in xdr_reserve_space()
1017 	xdr->buf->len += nbytes;  in xdr_reserve_space()
1034 int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, size_t nbytes)  in xdr_reserve_space_vec()  argument
1044 	if (xdr->iov == xdr->buf->head) {  in xdr_reserve_space_vec()
1045 		xdr->iov = NULL;  in xdr_reserve_space_vec()
1046 		xdr->end = xdr->p;  in xdr_reserve_space_vec()
1050 		thislen = xdr->buf->page_len % PAGE_SIZE;  in xdr_reserve_space_vec()
1053 		p = xdr_reserve_space(xdr, thislen);  in xdr_reserve_space_vec()
1086 void xdr_truncate_encode(struct xdr_stream *xdr, size_t len)  in xdr_truncate_encode()  argument
1088 	struct xdr_buf *buf = xdr->buf;  in xdr_truncate_encode()
1098 	xdr_commit_encode(xdr);  in xdr_truncate_encode()
1104 		xdr->p = tail->iov_base + tail->iov_len;  in xdr_truncate_encode()
1105 		WARN_ON_ONCE(!xdr->end);  in xdr_truncate_encode()
1106 		WARN_ON_ONCE(!xdr->iov);  in xdr_truncate_encode()
1116 	xdr->page_ptr = buf->pages + (new >> PAGE_SHIFT);  in xdr_truncate_encode()
1119 		xdr->p = page_address(*xdr->page_ptr);  in xdr_truncate_encode()
1120 		xdr->end = (void *)xdr->p + PAGE_SIZE;  in xdr_truncate_encode()
1121 		xdr->p = (void *)xdr->p + (new % PAGE_SIZE);  in xdr_truncate_encode()
1122 		WARN_ON_ONCE(xdr->iov);  in xdr_truncate_encode()
1126 		xdr->end = head->iov_base + head->iov_len;  in xdr_truncate_encode()
1128 	xdr->page_ptr--;  in xdr_truncate_encode()
1131 	xdr->p = head->iov_base + head->iov_len;  in xdr_truncate_encode()
1132 	xdr->iov = buf->head;  in xdr_truncate_encode()
1148 int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen)  in xdr_restrict_buflen()  argument
1150 	struct xdr_buf *buf = xdr->buf;  in xdr_restrict_buflen()
1151 	int left_in_this_buf = (void *)xdr->end - (void *)xdr->p;  in xdr_restrict_buflen()
1159 		xdr->end = (void *)xdr->end + newbuflen - end_offset;  in xdr_restrict_buflen()
1173 void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, unsigned int base,  in xdr_write_pages()  argument
1176 	struct xdr_buf *buf = xdr->buf;  in xdr_write_pages()
1182 	iov->iov_base = (char *)xdr->p;  in xdr_write_pages()
1184 	xdr->iov = iov;  in xdr_write_pages()
1189 		BUG_ON(xdr->p >= xdr->end);  in xdr_write_pages()
1190 		iov->iov_base = (char *)xdr->p + (len & 3);  in xdr_write_pages()
1193 		*xdr->p++ = 0;  in xdr_write_pages()
1200 static unsigned int xdr_set_iov(struct xdr_stream *xdr, struct kvec *iov,  in xdr_set_iov()  argument
1207 	xdr->p = (__be32*)(iov->iov_base + base);  in xdr_set_iov()
1208 	xdr->end = (__be32*)(iov->iov_base + len);  in xdr_set_iov()
1209 	xdr->iov = iov;  in xdr_set_iov()
1210 	xdr->page_ptr = NULL;  in xdr_set_iov()
1214 static unsigned int xdr_set_tail_base(struct xdr_stream *xdr,  in xdr_set_tail_base()  argument
1217 	struct xdr_buf *buf = xdr->buf;  in xdr_set_tail_base()
1219 	xdr_stream_set_pos(xdr, base + buf->page_len + buf->head->iov_len);  in xdr_set_tail_base()
1220 	return xdr_set_iov(xdr, buf->tail, base, len);  in xdr_set_tail_base()
1223 static unsigned int xdr_set_page_base(struct xdr_stream *xdr,  in xdr_set_page_base()  argument
1232 	maxlen = xdr->buf->page_len;  in xdr_set_page_base()
1240 	xdr_stream_page_set_pos(xdr, base);  in xdr_set_page_base()
1241 	base += xdr->buf->page_base;  in xdr_set_page_base()
1244 	xdr->page_ptr = &xdr->buf->pages[pgnr];  in xdr_set_page_base()
1245 	kaddr = page_address(*xdr->page_ptr);  in xdr_set_page_base()
1248 	xdr->p = (__be32*)(kaddr + pgoff);  in xdr_set_page_base()
1253 	xdr->end = (__be32*)(kaddr + pgend);  in xdr_set_page_base()
1254 	xdr->iov = NULL;  in xdr_set_page_base()
1258 static void xdr_set_page(struct xdr_stream *xdr, unsigned int base,  in xdr_set_page()  argument
1261 	if (xdr_set_page_base(xdr, base, len) == 0) {  in xdr_set_page()
1262 		base -= xdr->buf->page_len;  in xdr_set_page()
1263 		xdr_set_tail_base(xdr, base, len);  in xdr_set_page()
1267 static void xdr_set_next_page(struct xdr_stream *xdr)  in xdr_set_next_page()  argument
1271 	newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT;  in xdr_set_next_page()
1272 	newbase -= xdr->buf->page_base;  in xdr_set_next_page()
1273 	if (newbase < xdr->buf->page_len)  in xdr_set_next_page()
1274 		xdr_set_page_base(xdr, newbase, xdr_stream_remaining(xdr));  in xdr_set_next_page()
1276 		xdr_set_tail_base(xdr, 0, xdr_stream_remaining(xdr));  in xdr_set_next_page()
1279 static bool xdr_set_next_buffer(struct xdr_stream *xdr)  in xdr_set_next_buffer()  argument
1281 	if (xdr->page_ptr != NULL)  in xdr_set_next_buffer()
1282 		xdr_set_next_page(xdr);  in xdr_set_next_buffer()
1283 	else if (xdr->iov == xdr->buf->head)  in xdr_set_next_buffer()
1284 		xdr_set_page(xdr, 0, xdr_stream_remaining(xdr));  in xdr_set_next_buffer()
1285 	return xdr->p != xdr->end;  in xdr_set_next_buffer()
1295 void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,  in xdr_init_decode()  argument
1298 	xdr->buf = buf;  in xdr_init_decode()
1299 	xdr_reset_scratch_buffer(xdr);  in xdr_init_decode()
1300 	xdr->nwords = XDR_QUADLEN(buf->len);  in xdr_init_decode()
1301 	if (xdr_set_iov(xdr, buf->head, 0, buf->len) == 0 &&  in xdr_init_decode()
1302 	    xdr_set_page_base(xdr, 0, buf->len) == 0)  in xdr_init_decode()
1303 		xdr_set_iov(xdr, buf->tail, 0, buf->len);  in xdr_init_decode()
1304 	if (p != NULL && p > xdr->p && xdr->end >= p) {  in xdr_init_decode()
1305 		xdr->nwords -= p - xdr->p;  in xdr_init_decode()
1306 		xdr->p = p;  in xdr_init_decode()
1308 	xdr->rqst = rqst;  in xdr_init_decode()
1319 void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,  in xdr_init_decode_pages()  argument
1327 	xdr_init_decode(xdr, buf, NULL, NULL);  in xdr_init_decode_pages()
1331 static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)  in __xdr_inline_decode()  argument
1334 	__be32 *p = xdr->p;  in __xdr_inline_decode()
1337 	if (unlikely(nwords > xdr->nwords || q > xdr->end || q < p))  in __xdr_inline_decode()
1339 	xdr->p = q;  in __xdr_inline_decode()
1340 	xdr->nwords -= nwords;  in __xdr_inline_decode()
1344 static __be32 *xdr_copy_to_scratch(struct xdr_stream *xdr, size_t nbytes)  in xdr_copy_to_scratch()  argument
1347 	char *cpdest = xdr->scratch.iov_base;  in xdr_copy_to_scratch()
1348 	size_t cplen = (char *)xdr->end - (char *)xdr->p;  in xdr_copy_to_scratch()
1350 	if (nbytes > xdr->scratch.iov_len)  in xdr_copy_to_scratch()
1352 	p = __xdr_inline_decode(xdr, cplen);  in xdr_copy_to_scratch()
1356 	if (!xdr_set_next_buffer(xdr))  in xdr_copy_to_scratch()
1360 	p = __xdr_inline_decode(xdr, nbytes);  in xdr_copy_to_scratch()
1364 	return xdr->scratch.iov_base;  in xdr_copy_to_scratch()
1366 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_copy_to_scratch()
1380 __be32 * xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)  in xdr_inline_decode()  argument
1385 		return xdr->p;  in xdr_inline_decode()
1386 	if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))  in xdr_inline_decode()
1388 	p = __xdr_inline_decode(xdr, nbytes);  in xdr_inline_decode()
1391 	return xdr_copy_to_scratch(xdr, nbytes);  in xdr_inline_decode()
1393 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_inline_decode()
1398 static void xdr_realign_pages(struct xdr_stream *xdr)  in xdr_realign_pages()  argument
1400 	struct xdr_buf *buf = xdr->buf;  in xdr_realign_pages()
1402 	unsigned int cur = xdr_stream_pos(xdr);  in xdr_realign_pages()
1408 		trace_rpc_xdr_alignment(xdr, cur, copied);  in xdr_realign_pages()
1409 		xdr_set_page(xdr, 0, buf->page_len);  in xdr_realign_pages()
1413 static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_align_pages()  argument
1415 	struct xdr_buf *buf = xdr->buf;  in xdr_align_pages()
1419 	if (xdr->nwords == 0)  in xdr_align_pages()
1422 	xdr_realign_pages(xdr);  in xdr_align_pages()
1423 	if (nwords > xdr->nwords) {  in xdr_align_pages()
1424 		nwords = xdr->nwords;  in xdr_align_pages()
1429 	else if (nwords < xdr->nwords) {  in xdr_align_pages()
1432 		trace_rpc_xdr_alignment(xdr, len, copied);  in xdr_align_pages()
1449 unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_read_pages()  argument
1454 	pglen = xdr_align_pages(xdr, nwords << 2);  in xdr_read_pages()
1459 	end = xdr_stream_remaining(xdr) - pglen;  in xdr_read_pages()
1461 	xdr_set_tail_base(xdr, base, end);  in xdr_read_pages()
1466 unsigned int xdr_align_data(struct xdr_stream *xdr, unsigned int offset,  in xdr_align_data()  argument
1469 	struct xdr_buf *buf = xdr->buf;  in xdr_align_data()
1473 	xdr_realign_pages(xdr);  in xdr_align_data()
1474 	from = xdr_page_pos(xdr);  in xdr_align_data()
1496 	bytes = xdr_stream_remaining(xdr);  in xdr_align_data()
1501 	xdr->buf->len -= shift;  in xdr_align_data()
1502 	xdr_set_page(xdr, offset + length, bytes);  in xdr_align_data()
1507 unsigned int xdr_expand_hole(struct xdr_stream *xdr, unsigned int offset,  in xdr_expand_hole()  argument
1510 	struct xdr_buf *buf = xdr->buf;  in xdr_expand_hole()
1513 	xdr_realign_pages(xdr);  in xdr_expand_hole()
1514 	from = xdr_page_pos(xdr);  in xdr_expand_hole()
1523 		xdr_set_page(xdr, to, xdr_stream_remaining(xdr));  in xdr_expand_hole()
1525 		xdr_align_data(xdr, to, 0);  in xdr_expand_hole()
1542 void xdr_enter_page(struct xdr_stream *xdr, unsigned int len)  in xdr_enter_page()  argument
1544 	len = xdr_align_pages(xdr, len);  in xdr_enter_page()
1550 		xdr_set_page_base(xdr, 0, len);  in xdr_enter_page()
1642 bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf,  in xdr_stream_subsegment()  argument
1647 	if (xdr_buf_subsegment(xdr->buf, subbuf, xdr_stream_pos(xdr), nbytes))  in xdr_stream_subsegment()
1651 		if (!__xdr_inline_decode(xdr, subbuf->head[0].iov_len))  in xdr_stream_subsegment()
1659 		if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))  in xdr_stream_subsegment()
1661 		if (!__xdr_inline_decode(xdr, len))  in xdr_stream_subsegment()
2091 ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size)  in xdr_stream_decode_opaque()  argument
2096 	ret = xdr_stream_decode_opaque_inline(xdr, &p, size);  in xdr_stream_decode_opaque()
2117 ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr,  in xdr_stream_decode_opaque_dup()  argument
2123 	ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);  in xdr_stream_decode_opaque_dup()
2146 ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size)  in xdr_stream_decode_string()  argument
2151 	ret = xdr_stream_decode_opaque_inline(xdr, &p, size);  in xdr_stream_decode_string()
2175 ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,  in xdr_stream_decode_string_dup()  argument
2181 	ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);  in xdr_stream_decode_string_dup()