31 	unsigned int	quadlen = XDR_QUADLEN(obj->len);  in xdr_encode_netobj()
34 	*p++ = cpu_to_be32(obj->len);  in xdr_encode_netobj()
35 	memcpy(p, obj->data, obj->len);  in xdr_encode_netobj()
36 	return p + XDR_QUADLEN(obj->len);  in xdr_encode_netobj()
43 	unsigned int	len;  in xdr_decode_netobj()  local
45 	if ((len = be32_to_cpu(*p++)) > XDR_MAX_NETOBJ)  in xdr_decode_netobj()
47 	obj->len  = len;  in xdr_decode_netobj()
49 	return p + XDR_QUADLEN(len);  in xdr_decode_netobj()
109 	u32 len;  in xdr_decode_string_inplace()  local
111 	len = be32_to_cpu(*p++);  in xdr_decode_string_inplace()
112 	if (len > maxlen)  in xdr_decode_string_inplace()
114 	*lenp = len;  in xdr_decode_string_inplace()
116 	return p + XDR_QUADLEN(len);  in xdr_decode_string_inplace()
126 void xdr_terminate_string(const struct xdr_buf *buf, const u32 len)  in xdr_terminate_string()  argument
131 	kaddr[buf->page_base + len] = '\0';  in xdr_terminate_string()
179 		 struct page **pages, unsigned int base, unsigned int len)  in xdr_inline_pages()  argument
190 	xdr->page_len = len;  in xdr_inline_pages()
194 	xdr->buflen += len;  in xdr_inline_pages()
218 			size_t pgfrom_base, size_t len)  in _shift_data_left_pages()  argument
226 	if (!len)  in _shift_data_left_pages()
245 		copy = len;  in _shift_data_left_pages()
264 	} while ((len -= copy) != 0);  in _shift_data_left_pages()
283 		size_t pgfrom_base, size_t len)  in _shift_data_right_pages()  argument
291 	if (!len)  in _shift_data_right_pages()
294 	pgto_base += len;  in _shift_data_right_pages()
295 	pgfrom_base += len;  in _shift_data_right_pages()
314 		copy = len;  in _shift_data_right_pages()
332 	} while ((len -= copy) != 0);  in _shift_data_right_pages()
346 _copy_to_pages(struct page **pages, size_t pgbase, const char *p, size_t len)  in _copy_to_pages()  argument
352 	if (!len)  in _copy_to_pages()
360 		if (copy > len)  in _copy_to_pages()
361 			copy = len;  in _copy_to_pages()
367 		len -= copy;  in _copy_to_pages()
368 		if (len == 0)  in _copy_to_pages()
393 _copy_from_pages(char *p, struct page **pages, size_t pgbase, size_t len)  in _copy_from_pages()  argument
399 	if (!len)  in _copy_from_pages()
407 		if (copy > len)  in _copy_from_pages()
408 			copy = len;  in _copy_from_pages()
421 	} while ((len -= copy) != 0);  in _copy_from_pages()
426 			     unsigned int len)  in xdr_buf_iov_zero()  argument
430 	if (len > iov->iov_len - base)  in xdr_buf_iov_zero()
431 		len = iov->iov_len - base;  in xdr_buf_iov_zero()
432 	memset(iov->iov_base + base, 0, len);  in xdr_buf_iov_zero()
442 			       unsigned int len)  in xdr_buf_pages_zero()  argument
449 	if (!len)  in xdr_buf_pages_zero()
452 		xdr_buf_iov_zero(buf->tail, pgbase - buf->page_len, len);  in xdr_buf_pages_zero()
455 	if (pgbase + len > buf->page_len) {  in xdr_buf_pages_zero()
456 		xdr_buf_iov_zero(buf->tail, 0, pgbase + len - buf->page_len);  in xdr_buf_pages_zero()
457 		len = buf->page_len - pgbase;  in xdr_buf_pages_zero()
467 		if (zero > len)  in xdr_buf_pages_zero()
468 			zero = len;  in xdr_buf_pages_zero()
478 	} while ((len -= zero) != 0);  in xdr_buf_pages_zero()
509 static void xdr_buf_try_expand(struct xdr_buf *buf, unsigned int len)  in xdr_buf_try_expand()  argument
516 	if (sum > buf->len) {  in xdr_buf_try_expand()
517 		free_space = min_t(unsigned int, sum - buf->len, len);  in xdr_buf_try_expand()
518 		newlen = xdr_buf_pages_fill_sparse(buf, buf->len + free_space,  in xdr_buf_try_expand()
520 		free_space = newlen - buf->len;  in xdr_buf_try_expand()
521 		buf->len = newlen;  in xdr_buf_try_expand()
522 		len -= free_space;  in xdr_buf_try_expand()
523 		if (!len)  in xdr_buf_try_expand()
529 		free_space = min_t(unsigned int, buf->buflen - sum, len);  in xdr_buf_try_expand()
531 		buf->len += free_space;  in xdr_buf_try_expand()
536 				    unsigned int base, unsigned int len,  in xdr_buf_tail_copy_right()  argument
544 	if (len + to > tail->iov_len)  in xdr_buf_tail_copy_right()
545 		len = tail->iov_len - to;  in xdr_buf_tail_copy_right()
546 	memmove(tail->iov_base + to, tail->iov_base + base, len);  in xdr_buf_tail_copy_right()
550 				     unsigned int base, unsigned int len,  in xdr_buf_pages_copy_right()  argument
560 	if (len > buf->page_len - base)  in xdr_buf_pages_copy_right()
561 		len = buf->page_len - base;  in xdr_buf_pages_copy_right()
564 		if (tail->iov_len >= len + tato)  in xdr_buf_pages_copy_right()
565 			talen = len;  in xdr_buf_pages_copy_right()
568 	} else if (len + to >= buf->page_len) {  in xdr_buf_pages_copy_right()
570 		talen = len - pglen;  in xdr_buf_pages_copy_right()
574 		pglen = len;  in xdr_buf_pages_copy_right()
583 				    unsigned int base, unsigned int len,  in xdr_buf_head_copy_right()  argument
594 	if (len > head->iov_len - base)  in xdr_buf_head_copy_right()
595 		len = head->iov_len - base;  in xdr_buf_head_copy_right()
598 		talen = len;  in xdr_buf_head_copy_right()
601 		pglen = len;  in xdr_buf_head_copy_right()
607 		pglen = len - to;  in xdr_buf_head_copy_right()
614 	len -= talen;  in xdr_buf_head_copy_right()
615 	base += len;  in xdr_buf_head_copy_right()
620 	len -= pglen;  in xdr_buf_head_copy_right()
625 	base -= len;  in xdr_buf_head_copy_right()
626 	memmove(head->iov_base + to, head->iov_base + base, len);  in xdr_buf_head_copy_right()
630 				     unsigned int base, unsigned int len,  in xdr_buf_tail_shift_right()  argument
635 	if (base >= tail->iov_len || !shift || !len)  in xdr_buf_tail_shift_right()
637 	xdr_buf_tail_copy_right(buf, base, len, shift);  in xdr_buf_tail_shift_right()
641 				      unsigned int base, unsigned int len,  in xdr_buf_pages_shift_right()  argument
644 	if (!shift || !len)  in xdr_buf_pages_shift_right()
647 		xdr_buf_tail_shift_right(buf, base - buf->page_len, len, shift);  in xdr_buf_pages_shift_right()
650 	if (base + len > buf->page_len)  in xdr_buf_pages_shift_right()
651 		xdr_buf_tail_shift_right(buf, 0, base + len - buf->page_len,  in xdr_buf_pages_shift_right()
653 	xdr_buf_pages_copy_right(buf, base, len, shift);  in xdr_buf_pages_shift_right()
657 				     unsigned int base, unsigned int len,  in xdr_buf_head_shift_right()  argument
665 		xdr_buf_pages_shift_right(buf, head->iov_len - base, len,  in xdr_buf_head_shift_right()
669 	if (base + len > head->iov_len)  in xdr_buf_head_shift_right()
670 		xdr_buf_pages_shift_right(buf, 0, base + len - head->iov_len,  in xdr_buf_head_shift_right()
672 	xdr_buf_head_copy_right(buf, base, len, shift);  in xdr_buf_head_shift_right()
676 				   unsigned int len, unsigned int shift)  in xdr_buf_tail_copy_left()  argument
682 	if (len > tail->iov_len - base)  in xdr_buf_tail_copy_left()
683 		len = tail->iov_len - base;  in xdr_buf_tail_copy_left()
689 		unsigned int hdlen = len;  in xdr_buf_tail_copy_left()
698 		len -= hdlen;  in xdr_buf_tail_copy_left()
699 		if (!len)  in xdr_buf_tail_copy_left()
705 		unsigned int pglen = len;  in xdr_buf_tail_copy_left()
712 		len -= pglen;  in xdr_buf_tail_copy_left()
713 		if (!len)  in xdr_buf_tail_copy_left()
716 	memmove(tail->iov_base + base - shift, tail->iov_base + base, len);  in xdr_buf_tail_copy_left()
720 				    unsigned int base, unsigned int len,  in xdr_buf_pages_copy_left()  argument
727 	if (len > buf->page_len - base)  in xdr_buf_pages_copy_left()
728 		len = buf->page_len - base;  in xdr_buf_pages_copy_left()
733 		unsigned int hdlen = len;  in xdr_buf_pages_copy_left()
743 		len -= hdlen;  in xdr_buf_pages_copy_left()
744 		if (!len)  in xdr_buf_pages_copy_left()
749 			       buf->page_base + base, len);  in xdr_buf_pages_copy_left()
753 				    unsigned int base, unsigned int len,  in xdr_buf_tail_shift_left()  argument
756 	if (!shift || !len)  in xdr_buf_tail_shift_left()
758 	xdr_buf_tail_copy_left(buf, base, len, shift);  in xdr_buf_tail_shift_left()
762 				     unsigned int base, unsigned int len,  in xdr_buf_pages_shift_left()  argument
765 	if (!shift || !len)  in xdr_buf_pages_shift_left()
768 		xdr_buf_tail_shift_left(buf, base - buf->page_len, len, shift);  in xdr_buf_pages_shift_left()
771 	xdr_buf_pages_copy_left(buf, base, len, shift);  in xdr_buf_pages_shift_left()
772 	len += base;  in xdr_buf_pages_shift_left()
773 	if (len <= buf->page_len)  in xdr_buf_pages_shift_left()
775 	xdr_buf_tail_copy_left(buf, 0, len - buf->page_len, shift);  in xdr_buf_pages_shift_left()
787 static unsigned int xdr_shrink_bufhead(struct xdr_buf *buf, unsigned int len)  in xdr_shrink_bufhead()  argument
790 	unsigned int shift, buflen = max(buf->len, len);  in xdr_shrink_bufhead()
792 	WARN_ON_ONCE(len > head->iov_len);  in xdr_shrink_bufhead()
797 	if (len >= head->iov_len)  in xdr_shrink_bufhead()
799 	shift = head->iov_len - len;  in xdr_shrink_bufhead()
801 	xdr_buf_head_shift_right(buf, len, buflen - len, shift);  in xdr_shrink_bufhead()
802 	head->iov_len = len;  in xdr_shrink_bufhead()
804 	buf->len -= shift;  in xdr_shrink_bufhead()
816 static unsigned int xdr_shrink_pagelen(struct xdr_buf *buf, unsigned int len)  in xdr_shrink_pagelen()  argument
818 	unsigned int shift, buflen = buf->len - buf->head->iov_len;  in xdr_shrink_pagelen()
820 	WARN_ON_ONCE(len > buf->page_len);  in xdr_shrink_pagelen()
821 	if (buf->head->iov_len >= buf->len || len > buflen)  in xdr_shrink_pagelen()
822 		buflen = len;  in xdr_shrink_pagelen()
827 	if (len >= buf->page_len)  in xdr_shrink_pagelen()
829 	shift = buf->page_len - len;  in xdr_shrink_pagelen()
831 	xdr_buf_pages_shift_right(buf, len, buflen - len, shift);  in xdr_shrink_pagelen()
832 	buf->page_len = len;  in xdr_shrink_pagelen()
833 	buf->len -= shift;  in xdr_shrink_pagelen()
839 xdr_shift_buf(struct xdr_buf *buf, size_t len)  in xdr_shift_buf()  argument
841 	xdr_shrink_bufhead(buf, buf->head->iov_len - len);  in xdr_shift_buf()
851 	return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2;  in xdr_stream_pos()
857 	unsigned int blen = xdr->buf->len;  in xdr_stream_set_pos()
909 		size_t len;  in xdr_init_encode()  local
912 		len = (char *)p - (char *)xdr->p;  in xdr_init_encode()
914 		buf->len += len;  in xdr_init_encode()
915 		iov->iov_len += len;  in xdr_init_encode()
957 	if (xdr->buf->len + nbytes > xdr->buf->buflen)  in xdr_get_next_encode_buffer()
981 	space_left = xdr->buf->buflen - xdr->buf->len;  in xdr_get_next_encode_buffer()
984 	xdr->buf->len += nbytes;  in xdr_get_next_encode_buffer()
1017 	xdr->buf->len += nbytes;  in xdr_reserve_space()
1086 void xdr_truncate_encode(struct xdr_stream *xdr, size_t len)  in xdr_truncate_encode()  argument
1094 	if (len > buf->len) {  in xdr_truncate_encode()
1100 	fraglen = min_t(int, buf->len - len, tail->iov_len);  in xdr_truncate_encode()
1102 	buf->len -= fraglen;  in xdr_truncate_encode()
1110 	fraglen = min_t(int, buf->len - len, buf->page_len);  in xdr_truncate_encode()
1112 	buf->len -= fraglen;  in xdr_truncate_encode()
1129 	head->iov_len = len;  in xdr_truncate_encode()
1130 	buf->len = len;  in xdr_truncate_encode()
1152 	int end_offset = buf->len + left_in_this_buf;  in xdr_restrict_buflen()
1154 	if (newbuflen < 0 || newbuflen < buf->len)  in xdr_restrict_buflen()
1174 		 unsigned int len)  in xdr_write_pages()  argument
1180 	buf->page_len = len;  in xdr_write_pages()
1186 	if (len & 3) {  in xdr_write_pages()
1187 		unsigned int pad = 4 - (len & 3);  in xdr_write_pages()
1190 		iov->iov_base = (char *)xdr->p + (len & 3);  in xdr_write_pages()
1192 		len += pad;  in xdr_write_pages()
1195 	buf->buflen += len;  in xdr_write_pages()
1196 	buf->len += len;  in xdr_write_pages()
1201 				unsigned int base, unsigned int len)  in xdr_set_iov()  argument
1203 	if (len > iov->iov_len)  in xdr_set_iov()
1204 		len = iov->iov_len;  in xdr_set_iov()
1205 	if (unlikely(base > len))  in xdr_set_iov()
1206 		base = len;  in xdr_set_iov()
1208 	xdr->end = (__be32*)(iov->iov_base + len);  in xdr_set_iov()
1211 	return len - base;  in xdr_set_iov()
1215 				      unsigned int base, unsigned int len)  in xdr_set_tail_base()  argument
1220 	return xdr_set_iov(xdr, buf->tail, base, len);  in xdr_set_tail_base()
1224 				      unsigned int base, unsigned int len)  in xdr_set_page_base()  argument
1237 	if (len > maxlen)  in xdr_set_page_base()
1238 		len = maxlen;  in xdr_set_page_base()
1250 	pgend = pgoff + len;  in xdr_set_page_base()
1255 	return len;  in xdr_set_page_base()
1259 			 unsigned int len)  in xdr_set_page()  argument
1261 	if (xdr_set_page_base(xdr, base, len) == 0) {  in xdr_set_page()
1263 		xdr_set_tail_base(xdr, base, len);  in xdr_set_page()
1300 	xdr->nwords = XDR_QUADLEN(buf->len);  in xdr_init_decode()
1301 	if (xdr_set_iov(xdr, buf->head, 0, buf->len) == 0 &&  in xdr_init_decode()
1302 	    xdr_set_page_base(xdr, 0, buf->len) == 0)  in xdr_init_decode()
1303 		xdr_set_iov(xdr, buf->tail, 0, buf->len);  in xdr_init_decode()
1320 			   struct page **pages, unsigned int len)  in xdr_init_decode_pages()  argument
1324 	buf->page_len =  len;  in xdr_init_decode_pages()
1325 	buf->buflen =  len;  in xdr_init_decode_pages()
1326 	buf->len = len;  in xdr_init_decode_pages()
1413 static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_align_pages()  argument
1416 	unsigned int nwords = XDR_QUADLEN(len);  in xdr_align_pages()
1425 		len = nwords << 2;  in xdr_align_pages()
1427 	if (buf->page_len <= len)  in xdr_align_pages()
1428 		len = buf->page_len;  in xdr_align_pages()
1431 		copied = xdr_shrink_pagelen(buf, len);  in xdr_align_pages()
1432 		trace_rpc_xdr_alignment(xdr, len, copied);  in xdr_align_pages()
1434 	return len;  in xdr_align_pages()
1449 unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_read_pages()  argument
1451 	unsigned int nwords = XDR_QUADLEN(len);  in xdr_read_pages()
1462 	return len <= pglen ? len : pglen;  in xdr_read_pages()
1470 	unsigned int from, bytes, len;  in xdr_align_data()  local
1478 	if (from + buf->head->iov_len >= buf->len)  in xdr_align_data()
1481 	len = buf->len - buf->head->iov_len;  in xdr_align_data()
1494 	xdr_buf_pages_shift_left(buf, from, len, shift);  in xdr_align_data()
1501 	xdr->buf->len -= shift;  in xdr_align_data()
1519 		unsigned int buflen = buf->len - buf->head->iov_len;  in xdr_expand_hole()
1542 void xdr_enter_page(struct xdr_stream *xdr, unsigned int len)  in xdr_enter_page()  argument
1544 	len = xdr_align_pages(xdr, len);  in xdr_enter_page()
1549 	if (len != 0)  in xdr_enter_page()
1550 		xdr_set_page_base(xdr, 0, len);  in xdr_enter_page()
1561 	buf->buflen = buf->len = iov->iov_len;  in xdr_buf_from_iov()
1580 		       unsigned int base, unsigned int len)  in xdr_buf_subsegment()  argument
1582 	subbuf->buflen = subbuf->len = len;  in xdr_buf_subsegment()
1585 		subbuf->head[0].iov_len = min_t(unsigned int, len,  in xdr_buf_subsegment()
1587 		len -= subbuf->head[0].iov_len;  in xdr_buf_subsegment()
1596 		subbuf->page_len = min(buf->page_len - base, len);  in xdr_buf_subsegment()
1600 		len -= subbuf->page_len;  in xdr_buf_subsegment()
1611 		subbuf->tail[0].iov_len = min_t(unsigned int, len,  in xdr_buf_subsegment()
1613 		len -= subbuf->tail[0].iov_len;  in xdr_buf_subsegment()
1621 	if (base || len)  in xdr_buf_subsegment()
1645 	unsigned int remaining, offset, len;  in xdr_stream_subsegment()  local
1657 		len = min_t(unsigned int, remaining, PAGE_SIZE) - offset;  in xdr_stream_subsegment()
1661 		if (!__xdr_inline_decode(xdr, len))  in xdr_stream_subsegment()
1664 		remaining -= len;  in xdr_stream_subsegment()
1682 void xdr_buf_trim(struct xdr_buf *buf, unsigned int len)  in xdr_buf_trim()  argument
1685 	unsigned int trim = len;  in xdr_buf_trim()
1709 	buf->len -= (len - trim);  in xdr_buf_trim()
1714 				      void *obj, unsigned int len)  in __read_bytes_from_xdr_buf()  argument
1718 	this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);  in __read_bytes_from_xdr_buf()
1720 	len -= this_len;  in __read_bytes_from_xdr_buf()
1722 	this_len = min_t(unsigned int, len, subbuf->page_len);  in __read_bytes_from_xdr_buf()
1724 	len -= this_len;  in __read_bytes_from_xdr_buf()
1726 	this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);  in __read_bytes_from_xdr_buf()
1732 			    void *obj, unsigned int len)  in read_bytes_from_xdr_buf()  argument
1737 	status = xdr_buf_subsegment(buf, &subbuf, base, len);  in read_bytes_from_xdr_buf()
1740 	__read_bytes_from_xdr_buf(&subbuf, obj, len);  in read_bytes_from_xdr_buf()
1746 				     void *obj, unsigned int len)  in __write_bytes_to_xdr_buf()  argument
1750 	this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);  in __write_bytes_to_xdr_buf()
1752 	len -= this_len;  in __write_bytes_to_xdr_buf()
1754 	this_len = min_t(unsigned int, len, subbuf->page_len);  in __write_bytes_to_xdr_buf()
1756 	len -= this_len;  in __write_bytes_to_xdr_buf()
1758 	this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);  in __write_bytes_to_xdr_buf()
1764 			   void *obj, unsigned int len)  in write_bytes_to_xdr_buf()  argument
1769 	status = xdr_buf_subsegment(buf, &subbuf, base, len);  in write_bytes_to_xdr_buf()
1772 	__write_bytes_to_xdr_buf(&subbuf, obj, len);  in write_bytes_to_xdr_buf()
1814 				    desc->elem_size > buf->len)  in xdr_xcode_array2()
1995 	if (base >= buf->len)  in xdr_decode_array2()
2014 		    unsigned int len,  in xdr_process_buf()  argument
2027 		if (thislen > len)  in xdr_process_buf()
2028 			thislen = len;  in xdr_process_buf()
2034 		len -= thislen;  in xdr_process_buf()
2036 	if (len == 0)  in xdr_process_buf()
2043 		if (page_len > len)  in xdr_process_buf()
2044 			page_len = len;  in xdr_process_buf()
2045 		len -= page_len;  in xdr_process_buf()
2063 	if (len == 0)  in xdr_process_buf()
2067 		if (thislen > len)  in xdr_process_buf()
2068 			thislen = len;  in xdr_process_buf()
2071 		len -= thislen;  in xdr_process_buf()
2073 	if (len != 0)  in xdr_process_buf()