836 static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_cursor_init()  argument
839 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_cursor_init()
840 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_cursor_init()
842 	cursor->resid = min_t(size_t, length, data->bio_length);  in ceph_msg_data_bio_cursor_init()
844 	if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_cursor_init()
845 		it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_cursor_init()
847 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_cursor_init()
848 	cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter);  in ceph_msg_data_bio_cursor_init()
851 static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_next()  argument
855 	struct bio_vec bv = bio_iter_iovec(cursor->bio_iter.bio,  in ceph_msg_data_bio_next()
856 					   cursor->bio_iter.iter);  in ceph_msg_data_bio_next()
863 static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_advance()  argument
866 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_advance()
868 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bio_advance()
870 	cursor->resid -= bytes;  in ceph_msg_data_bio_advance()
873 	if (!cursor->resid) {  in ceph_msg_data_bio_advance()
874 		BUG_ON(!cursor->last_piece);  in ceph_msg_data_bio_advance()
884 		if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_advance()
885 			it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_advance()
888 	BUG_ON(cursor->last_piece);  in ceph_msg_data_bio_advance()
889 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_advance()
890 	cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter);  in ceph_msg_data_bio_advance()
895 static void ceph_msg_data_bvecs_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_cursor_init()  argument
898 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bvecs_cursor_init()
901 	cursor->resid = min_t(size_t, length, data->bvec_pos.iter.bi_size);  in ceph_msg_data_bvecs_cursor_init()
902 	cursor->bvec_iter = data->bvec_pos.iter;  in ceph_msg_data_bvecs_cursor_init()
903 	cursor->bvec_iter.bi_size = cursor->resid;  in ceph_msg_data_bvecs_cursor_init()
905 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_cursor_init()
906 	cursor->last_piece =  in ceph_msg_data_bvecs_cursor_init()
907 	    cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter);  in ceph_msg_data_bvecs_cursor_init()
910 static struct page *ceph_msg_data_bvecs_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_next()  argument
914 	struct bio_vec bv = bvec_iter_bvec(cursor->data->bvec_pos.bvecs,  in ceph_msg_data_bvecs_next()
915 					   cursor->bvec_iter);  in ceph_msg_data_bvecs_next()
922 static bool ceph_msg_data_bvecs_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_advance()  argument
925 	struct bio_vec *bvecs = cursor->data->bvec_pos.bvecs;  in ceph_msg_data_bvecs_advance()
927 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bvecs_advance()
928 	BUG_ON(bytes > bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
929 	cursor->resid -= bytes;  in ceph_msg_data_bvecs_advance()
930 	bvec_iter_advance(bvecs, &cursor->bvec_iter, bytes);  in ceph_msg_data_bvecs_advance()
932 	if (!cursor->resid) {  in ceph_msg_data_bvecs_advance()
933 		BUG_ON(!cursor->last_piece);  in ceph_msg_data_bvecs_advance()
937 	if (!bytes || cursor->bvec_iter.bi_bvec_done)  in ceph_msg_data_bvecs_advance()
940 	BUG_ON(cursor->last_piece);  in ceph_msg_data_bvecs_advance()
941 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
942 	cursor->last_piece =  in ceph_msg_data_bvecs_advance()
943 	    cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter);  in ceph_msg_data_bvecs_advance()
951 static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_cursor_init()  argument
954 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_cursor_init()
962 	cursor->resid = min(length, data->length);  in ceph_msg_data_pages_cursor_init()
964 	cursor->page_offset = data->alignment & ~PAGE_MASK;  in ceph_msg_data_pages_cursor_init()
965 	cursor->page_index = 0;  in ceph_msg_data_pages_cursor_init()
967 	cursor->page_count = (unsigned short)page_count;  in ceph_msg_data_pages_cursor_init()
968 	BUG_ON(length > SIZE_MAX - cursor->page_offset);  in ceph_msg_data_pages_cursor_init()
969 	cursor->last_piece = cursor->page_offset + cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_cursor_init()
973 ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_next()  argument
976 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_next()
980 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_next()
981 	BUG_ON(cursor->page_offset >= PAGE_SIZE);  in ceph_msg_data_pages_next()
983 	*page_offset = cursor->page_offset;  in ceph_msg_data_pages_next()
984 	if (cursor->last_piece)  in ceph_msg_data_pages_next()
985 		*length = cursor->resid;  in ceph_msg_data_pages_next()
989 	return data->pages[cursor->page_index];  in ceph_msg_data_pages_next()
992 static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_advance()  argument
995 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_PAGES);  in ceph_msg_data_pages_advance()
997 	BUG_ON(cursor->page_offset + bytes > PAGE_SIZE);  in ceph_msg_data_pages_advance()
1001 	cursor->resid -= bytes;  in ceph_msg_data_pages_advance()
1002 	cursor->page_offset = (cursor->page_offset + bytes) & ~PAGE_MASK;  in ceph_msg_data_pages_advance()
1003 	if (!bytes || cursor->page_offset)  in ceph_msg_data_pages_advance()
1006 	if (!cursor->resid)  in ceph_msg_data_pages_advance()
1011 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_advance()
1012 	cursor->page_index++;  in ceph_msg_data_pages_advance()
1013 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_advance()
1023 ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_cursor_init()  argument
1026 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_cursor_init()
1041 	cursor->resid = min(length, pagelist->length);  in ceph_msg_data_pagelist_cursor_init()
1042 	cursor->page = page;  in ceph_msg_data_pagelist_cursor_init()
1043 	cursor->offset = 0;  in ceph_msg_data_pagelist_cursor_init()
1044 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_cursor_init()
1048 ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_next()  argument
1051 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_next()
1059 	BUG_ON(!cursor->page);  in ceph_msg_data_pagelist_next()
1060 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_next()
1063 	*page_offset = cursor->offset & ~PAGE_MASK;  in ceph_msg_data_pagelist_next()
1064 	if (cursor->last_piece)  in ceph_msg_data_pagelist_next()
1065 		*length = cursor->resid;  in ceph_msg_data_pagelist_next()
1069 	return cursor->page;  in ceph_msg_data_pagelist_next()
1072 static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_advance()  argument
1075 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_advance()
1083 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_advance()
1084 	BUG_ON((cursor->offset & ~PAGE_MASK) + bytes > PAGE_SIZE);  in ceph_msg_data_pagelist_advance()
1088 	cursor->resid -= bytes;  in ceph_msg_data_pagelist_advance()
1089 	cursor->offset += bytes;  in ceph_msg_data_pagelist_advance()
1091 	if (!bytes || cursor->offset & ~PAGE_MASK)  in ceph_msg_data_pagelist_advance()
1094 	if (!cursor->resid)  in ceph_msg_data_pagelist_advance()
1099 	BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));  in ceph_msg_data_pagelist_advance()
1100 	cursor->page = list_next_entry(cursor->page, lru);  in ceph_msg_data_pagelist_advance()
1101 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_advance()
1114 static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)  in __ceph_msg_data_cursor_init()  argument
1116 	size_t length = cursor->total_resid;  in __ceph_msg_data_cursor_init()
1118 	switch (cursor->data->type) {  in __ceph_msg_data_cursor_init()
1120 		ceph_msg_data_pagelist_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1123 		ceph_msg_data_pages_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1127 		ceph_msg_data_bio_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1131 		ceph_msg_data_bvecs_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1138 	cursor->need_crc = true;  in __ceph_msg_data_cursor_init()
1143 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in ceph_msg_data_cursor_init()  local
1150 	cursor->data_head = &msg->data;  in ceph_msg_data_cursor_init()
1151 	cursor->total_resid = length;  in ceph_msg_data_cursor_init()
1153 	cursor->data = data;  in ceph_msg_data_cursor_init()
1155 	__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_cursor_init()
1163 static struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_next()  argument
1169 	switch (cursor->data->type) {  in ceph_msg_data_next()
1171 		page = ceph_msg_data_pagelist_next(cursor, page_offset, length);  in ceph_msg_data_next()
1174 		page = ceph_msg_data_pages_next(cursor, page_offset, length);  in ceph_msg_data_next()
1178 		page = ceph_msg_data_bio_next(cursor, page_offset, length);  in ceph_msg_data_next()
1182 		page = ceph_msg_data_bvecs_next(cursor, page_offset, length);  in ceph_msg_data_next()
1193 	BUG_ON(*length > cursor->resid);  in ceph_msg_data_next()
1195 		*last_piece = cursor->last_piece;  in ceph_msg_data_next()
1204 static void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_advance()  argument
1209 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_advance()
1210 	switch (cursor->data->type) {  in ceph_msg_data_advance()
1212 		new_piece = ceph_msg_data_pagelist_advance(cursor, bytes);  in ceph_msg_data_advance()
1215 		new_piece = ceph_msg_data_pages_advance(cursor, bytes);  in ceph_msg_data_advance()
1219 		new_piece = ceph_msg_data_bio_advance(cursor, bytes);  in ceph_msg_data_advance()
1223 		new_piece = ceph_msg_data_bvecs_advance(cursor, bytes);  in ceph_msg_data_advance()
1230 	cursor->total_resid -= bytes;  in ceph_msg_data_advance()
1232 	if (!cursor->resid && cursor->total_resid) {  in ceph_msg_data_advance()
1233 		WARN_ON(!cursor->last_piece);  in ceph_msg_data_advance()
1234 		BUG_ON(list_is_last(&cursor->data->links, cursor->data_head));  in ceph_msg_data_advance()
1235 		cursor->data = list_next_entry(cursor->data, links);  in ceph_msg_data_advance()
1236 		__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_advance()
1239 	cursor->need_crc = new_piece;  in ceph_msg_data_advance()
1587 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in write_partial_message_data()  local
1605 	while (cursor->total_resid) {  in write_partial_message_data()
1612 		if (!cursor->resid) {  in write_partial_message_data()
1613 			ceph_msg_data_advance(cursor, 0);  in write_partial_message_data()
1617 		page = ceph_msg_data_next(cursor, &page_offset, &length,  in write_partial_message_data()
1627 		if (do_datacrc && cursor->need_crc)  in write_partial_message_data()
1629 		ceph_msg_data_advance(cursor, (size_t)ret);  in write_partial_message_data()
2342 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in read_partial_msg_data()  local
2356 	while (cursor->total_resid) {  in read_partial_msg_data()
2357 		if (!cursor->resid) {  in read_partial_msg_data()
2358 			ceph_msg_data_advance(cursor, 0);  in read_partial_msg_data()
2362 		page = ceph_msg_data_next(cursor, &page_offset, &length, NULL);  in read_partial_msg_data()
2373 		ceph_msg_data_advance(cursor, (size_t)ret);  in read_partial_msg_data()
3177 			con->out_skip += msg->cursor.total_resid;  in ceph_msg_revoke()