814 static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_cursor_init()  argument
817 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_cursor_init()
818 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_cursor_init()
820 	cursor->resid = min_t(size_t, length, data->bio_length);  in ceph_msg_data_bio_cursor_init()
822 	if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_cursor_init()
823 		it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_cursor_init()
825 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_cursor_init()
826 	cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter);  in ceph_msg_data_bio_cursor_init()
829 static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_next()  argument
833 	struct bio_vec bv = bio_iter_iovec(cursor->bio_iter.bio,  in ceph_msg_data_bio_next()
834 					   cursor->bio_iter.iter);  in ceph_msg_data_bio_next()
841 static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_advance()  argument
844 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_advance()
847 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bio_advance()
849 	cursor->resid -= bytes;  in ceph_msg_data_bio_advance()
852 	if (!cursor->resid) {  in ceph_msg_data_bio_advance()
853 		BUG_ON(!cursor->last_piece);  in ceph_msg_data_bio_advance()
864 		if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_advance()
865 			it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_advance()
868 	BUG_ON(cursor->last_piece);  in ceph_msg_data_bio_advance()
869 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_advance()
870 	cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter);  in ceph_msg_data_bio_advance()
875 static void ceph_msg_data_bvecs_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_cursor_init()  argument
878 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bvecs_cursor_init()
881 	cursor->resid = min_t(size_t, length, data->bvec_pos.iter.bi_size);  in ceph_msg_data_bvecs_cursor_init()
882 	cursor->bvec_iter = data->bvec_pos.iter;  in ceph_msg_data_bvecs_cursor_init()
883 	cursor->bvec_iter.bi_size = cursor->resid;  in ceph_msg_data_bvecs_cursor_init()
885 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_cursor_init()
886 	cursor->last_piece =  in ceph_msg_data_bvecs_cursor_init()
887 	    cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter);  in ceph_msg_data_bvecs_cursor_init()
890 static struct page *ceph_msg_data_bvecs_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_next()  argument
894 	struct bio_vec bv = bvec_iter_bvec(cursor->data->bvec_pos.bvecs,  in ceph_msg_data_bvecs_next()
895 					   cursor->bvec_iter);  in ceph_msg_data_bvecs_next()
902 static bool ceph_msg_data_bvecs_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_advance()  argument
905 	struct bio_vec *bvecs = cursor->data->bvec_pos.bvecs;  in ceph_msg_data_bvecs_advance()
906 	struct page *page = bvec_iter_page(bvecs, cursor->bvec_iter);  in ceph_msg_data_bvecs_advance()
908 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bvecs_advance()
909 	BUG_ON(bytes > bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
910 	cursor->resid -= bytes;  in ceph_msg_data_bvecs_advance()
911 	bvec_iter_advance(bvecs, &cursor->bvec_iter, bytes);  in ceph_msg_data_bvecs_advance()
913 	if (!cursor->resid) {  in ceph_msg_data_bvecs_advance()
914 		BUG_ON(!cursor->last_piece);  in ceph_msg_data_bvecs_advance()
918 	if (!bytes || (cursor->bvec_iter.bi_bvec_done &&  in ceph_msg_data_bvecs_advance()
919 		       page == bvec_iter_page(bvecs, cursor->bvec_iter)))  in ceph_msg_data_bvecs_advance()
922 	BUG_ON(cursor->last_piece);  in ceph_msg_data_bvecs_advance()
923 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
924 	cursor->last_piece =  in ceph_msg_data_bvecs_advance()
925 	    cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter);  in ceph_msg_data_bvecs_advance()
933 static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_cursor_init()  argument
936 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_cursor_init()
944 	cursor->resid = min(length, data->length);  in ceph_msg_data_pages_cursor_init()
946 	cursor->page_offset = data->alignment & ~PAGE_MASK;  in ceph_msg_data_pages_cursor_init()
947 	cursor->page_index = 0;  in ceph_msg_data_pages_cursor_init()
949 	cursor->page_count = (unsigned short)page_count;  in ceph_msg_data_pages_cursor_init()
950 	BUG_ON(length > SIZE_MAX - cursor->page_offset);  in ceph_msg_data_pages_cursor_init()
951 	cursor->last_piece = cursor->page_offset + cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_cursor_init()
955 ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_next()  argument
958 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_next()
962 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_next()
963 	BUG_ON(cursor->page_offset >= PAGE_SIZE);  in ceph_msg_data_pages_next()
965 	*page_offset = cursor->page_offset;  in ceph_msg_data_pages_next()
966 	if (cursor->last_piece)  in ceph_msg_data_pages_next()
967 		*length = cursor->resid;  in ceph_msg_data_pages_next()
971 	return data->pages[cursor->page_index];  in ceph_msg_data_pages_next()
974 static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_advance()  argument
977 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_PAGES);  in ceph_msg_data_pages_advance()
979 	BUG_ON(cursor->page_offset + bytes > PAGE_SIZE);  in ceph_msg_data_pages_advance()
983 	cursor->resid -= bytes;  in ceph_msg_data_pages_advance()
984 	cursor->page_offset = (cursor->page_offset + bytes) & ~PAGE_MASK;  in ceph_msg_data_pages_advance()
985 	if (!bytes || cursor->page_offset)  in ceph_msg_data_pages_advance()
988 	if (!cursor->resid)  in ceph_msg_data_pages_advance()
993 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_advance()
994 	cursor->page_index++;  in ceph_msg_data_pages_advance()
995 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_advance()
1005 ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_cursor_init()  argument
1008 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_cursor_init()
1023 	cursor->resid = min(length, pagelist->length);  in ceph_msg_data_pagelist_cursor_init()
1024 	cursor->page = page;  in ceph_msg_data_pagelist_cursor_init()
1025 	cursor->offset = 0;  in ceph_msg_data_pagelist_cursor_init()
1026 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_cursor_init()
1030 ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_next()  argument
1033 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_next()
1041 	BUG_ON(!cursor->page);  in ceph_msg_data_pagelist_next()
1042 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_next()
1045 	*page_offset = cursor->offset & ~PAGE_MASK;  in ceph_msg_data_pagelist_next()
1046 	if (cursor->last_piece)  in ceph_msg_data_pagelist_next()
1047 		*length = cursor->resid;  in ceph_msg_data_pagelist_next()
1051 	return cursor->page;  in ceph_msg_data_pagelist_next()
1054 static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_advance()  argument
1057 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_advance()
1065 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_advance()
1066 	BUG_ON((cursor->offset & ~PAGE_MASK) + bytes > PAGE_SIZE);  in ceph_msg_data_pagelist_advance()
1070 	cursor->resid -= bytes;  in ceph_msg_data_pagelist_advance()
1071 	cursor->offset += bytes;  in ceph_msg_data_pagelist_advance()
1073 	if (!bytes || cursor->offset & ~PAGE_MASK)  in ceph_msg_data_pagelist_advance()
1076 	if (!cursor->resid)  in ceph_msg_data_pagelist_advance()
1081 	BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));  in ceph_msg_data_pagelist_advance()
1082 	cursor->page = list_next_entry(cursor->page, lru);  in ceph_msg_data_pagelist_advance()
1083 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_advance()
1096 static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)  in __ceph_msg_data_cursor_init()  argument
1098 	size_t length = cursor->total_resid;  in __ceph_msg_data_cursor_init()
1100 	switch (cursor->data->type) {  in __ceph_msg_data_cursor_init()
1102 		ceph_msg_data_pagelist_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1105 		ceph_msg_data_pages_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1109 		ceph_msg_data_bio_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1113 		ceph_msg_data_bvecs_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1120 	cursor->need_crc = true;  in __ceph_msg_data_cursor_init()
1125 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in ceph_msg_data_cursor_init()  local
1131 	cursor->total_resid = length;  in ceph_msg_data_cursor_init()
1132 	cursor->data = msg->data;  in ceph_msg_data_cursor_init()
1134 	__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_cursor_init()
1142 static struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_next()  argument
1148 	switch (cursor->data->type) {  in ceph_msg_data_next()
1150 		page = ceph_msg_data_pagelist_next(cursor, page_offset, length);  in ceph_msg_data_next()
1153 		page = ceph_msg_data_pages_next(cursor, page_offset, length);  in ceph_msg_data_next()
1157 		page = ceph_msg_data_bio_next(cursor, page_offset, length);  in ceph_msg_data_next()
1161 		page = ceph_msg_data_bvecs_next(cursor, page_offset, length);  in ceph_msg_data_next()
1172 	BUG_ON(*length > cursor->resid);  in ceph_msg_data_next()
1174 		*last_piece = cursor->last_piece;  in ceph_msg_data_next()
1183 static void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_advance()  argument
1188 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_advance()
1189 	switch (cursor->data->type) {  in ceph_msg_data_advance()
1191 		new_piece = ceph_msg_data_pagelist_advance(cursor, bytes);  in ceph_msg_data_advance()
1194 		new_piece = ceph_msg_data_pages_advance(cursor, bytes);  in ceph_msg_data_advance()
1198 		new_piece = ceph_msg_data_bio_advance(cursor, bytes);  in ceph_msg_data_advance()
1202 		new_piece = ceph_msg_data_bvecs_advance(cursor, bytes);  in ceph_msg_data_advance()
1209 	cursor->total_resid -= bytes;  in ceph_msg_data_advance()
1211 	if (!cursor->resid && cursor->total_resid) {  in ceph_msg_data_advance()
1212 		WARN_ON(!cursor->last_piece);  in ceph_msg_data_advance()
1213 		cursor->data++;  in ceph_msg_data_advance()
1214 		__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_advance()
1217 	cursor->need_crc = new_piece;  in ceph_msg_data_advance()
1562 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in write_partial_message_data()  local
1581 	while (cursor->total_resid) {  in write_partial_message_data()
1587 		if (!cursor->resid) {  in write_partial_message_data()
1588 			ceph_msg_data_advance(cursor, 0);  in write_partial_message_data()
1592 		page = ceph_msg_data_next(cursor, &page_offset, &length, NULL);  in write_partial_message_data()
1593 		if (length == cursor->total_resid)  in write_partial_message_data()
1603 		if (do_datacrc && cursor->need_crc)  in write_partial_message_data()
1605 		ceph_msg_data_advance(cursor, (size_t)ret);  in write_partial_message_data()
2322 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in read_partial_msg_data()  local
2335 	while (cursor->total_resid) {  in read_partial_msg_data()
2336 		if (!cursor->resid) {  in read_partial_msg_data()
2337 			ceph_msg_data_advance(cursor, 0);  in read_partial_msg_data()
2341 		page = ceph_msg_data_next(cursor, &page_offset, &length, NULL);  in read_partial_msg_data()
2352 		ceph_msg_data_advance(cursor, (size_t)ret);  in read_partial_msg_data()
3162 			con->out_skip += msg->cursor.total_resid;  in ceph_msg_revoke()