719 static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_cursor_init()  argument
722 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_cursor_init()
723 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_cursor_init()
725 	cursor->resid = min_t(size_t, length, data->bio_length);  in ceph_msg_data_bio_cursor_init()
727 	if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_cursor_init()
728 		it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_cursor_init()
730 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_cursor_init()
733 static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_next()  argument
737 	struct bio_vec bv = bio_iter_iovec(cursor->bio_iter.bio,  in ceph_msg_data_bio_next()
738 					   cursor->bio_iter.iter);  in ceph_msg_data_bio_next()
745 static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_advance()  argument
748 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_advance()
751 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bio_advance()
753 	cursor->resid -= bytes;  in ceph_msg_data_bio_advance()
756 	if (!cursor->resid)  in ceph_msg_data_bio_advance()
766 		if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_advance()
767 			it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_advance()
770 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_advance()
775 static void ceph_msg_data_bvecs_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_cursor_init()  argument
778 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bvecs_cursor_init()
781 	cursor->resid = min_t(size_t, length, data->bvec_pos.iter.bi_size);  in ceph_msg_data_bvecs_cursor_init()
782 	cursor->bvec_iter = data->bvec_pos.iter;  in ceph_msg_data_bvecs_cursor_init()
783 	cursor->bvec_iter.bi_size = cursor->resid;  in ceph_msg_data_bvecs_cursor_init()
785 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_cursor_init()
788 static struct page *ceph_msg_data_bvecs_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_next()  argument
792 	struct bio_vec bv = bvec_iter_bvec(cursor->data->bvec_pos.bvecs,  in ceph_msg_data_bvecs_next()
793 					   cursor->bvec_iter);  in ceph_msg_data_bvecs_next()
800 static bool ceph_msg_data_bvecs_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_advance()  argument
803 	struct bio_vec *bvecs = cursor->data->bvec_pos.bvecs;  in ceph_msg_data_bvecs_advance()
804 	struct page *page = bvec_iter_page(bvecs, cursor->bvec_iter);  in ceph_msg_data_bvecs_advance()
806 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bvecs_advance()
807 	BUG_ON(bytes > bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
808 	cursor->resid -= bytes;  in ceph_msg_data_bvecs_advance()
809 	bvec_iter_advance(bvecs, &cursor->bvec_iter, bytes);  in ceph_msg_data_bvecs_advance()
811 	if (!cursor->resid)  in ceph_msg_data_bvecs_advance()
814 	if (!bytes || (cursor->bvec_iter.bi_bvec_done &&  in ceph_msg_data_bvecs_advance()
815 		       page == bvec_iter_page(bvecs, cursor->bvec_iter)))  in ceph_msg_data_bvecs_advance()
818 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
826 static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_cursor_init()  argument
829 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_cursor_init()
837 	cursor->resid = min(length, data->length);  in ceph_msg_data_pages_cursor_init()
839 	cursor->page_offset = data->alignment & ~PAGE_MASK;  in ceph_msg_data_pages_cursor_init()
840 	cursor->page_index = 0;  in ceph_msg_data_pages_cursor_init()
842 	cursor->page_count = (unsigned short)page_count;  in ceph_msg_data_pages_cursor_init()
843 	BUG_ON(length > SIZE_MAX - cursor->page_offset);  in ceph_msg_data_pages_cursor_init()
847 ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_next()  argument
850 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_next()
854 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_next()
855 	BUG_ON(cursor->page_offset >= PAGE_SIZE);  in ceph_msg_data_pages_next()
857 	*page_offset = cursor->page_offset;  in ceph_msg_data_pages_next()
858 	*length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset);  in ceph_msg_data_pages_next()
859 	return data->pages[cursor->page_index];  in ceph_msg_data_pages_next()
862 static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_advance()  argument
865 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_PAGES);  in ceph_msg_data_pages_advance()
867 	BUG_ON(cursor->page_offset + bytes > PAGE_SIZE);  in ceph_msg_data_pages_advance()
871 	cursor->resid -= bytes;  in ceph_msg_data_pages_advance()
872 	cursor->page_offset = (cursor->page_offset + bytes) & ~PAGE_MASK;  in ceph_msg_data_pages_advance()
873 	if (!bytes || cursor->page_offset)  in ceph_msg_data_pages_advance()
876 	if (!cursor->resid)  in ceph_msg_data_pages_advance()
881 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_advance()
882 	cursor->page_index++;  in ceph_msg_data_pages_advance()
891 ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_cursor_init()  argument
894 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_cursor_init()
909 	cursor->resid = min(length, pagelist->length);  in ceph_msg_data_pagelist_cursor_init()
910 	cursor->page = page;  in ceph_msg_data_pagelist_cursor_init()
911 	cursor->offset = 0;  in ceph_msg_data_pagelist_cursor_init()
915 ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_next()  argument
918 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_next()
926 	BUG_ON(!cursor->page);  in ceph_msg_data_pagelist_next()
927 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_next()
930 	*page_offset = cursor->offset & ~PAGE_MASK;  in ceph_msg_data_pagelist_next()
931 	*length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset);  in ceph_msg_data_pagelist_next()
932 	return cursor->page;  in ceph_msg_data_pagelist_next()
935 static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_advance()  argument
938 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_advance()
946 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_advance()
947 	BUG_ON((cursor->offset & ~PAGE_MASK) + bytes > PAGE_SIZE);  in ceph_msg_data_pagelist_advance()
951 	cursor->resid -= bytes;  in ceph_msg_data_pagelist_advance()
952 	cursor->offset += bytes;  in ceph_msg_data_pagelist_advance()
954 	if (!bytes || cursor->offset & ~PAGE_MASK)  in ceph_msg_data_pagelist_advance()
957 	if (!cursor->resid)  in ceph_msg_data_pagelist_advance()
962 	BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));  in ceph_msg_data_pagelist_advance()
963 	cursor->page = list_next_entry(cursor->page, lru);  in ceph_msg_data_pagelist_advance()
975 static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)  in __ceph_msg_data_cursor_init()  argument
977 	size_t length = cursor->total_resid;  in __ceph_msg_data_cursor_init()
979 	switch (cursor->data->type) {  in __ceph_msg_data_cursor_init()
981 		ceph_msg_data_pagelist_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
984 		ceph_msg_data_pages_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
988 		ceph_msg_data_bio_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
992 		ceph_msg_data_bvecs_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
999 	cursor->need_crc = true;  in __ceph_msg_data_cursor_init()
1002 void ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_cursor_init()  argument
1009 	cursor->total_resid = length;  in ceph_msg_data_cursor_init()
1010 	cursor->data = msg->data;  in ceph_msg_data_cursor_init()
1012 	__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_cursor_init()
1020 struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_next()  argument
1025 	switch (cursor->data->type) {  in ceph_msg_data_next()
1027 		page = ceph_msg_data_pagelist_next(cursor, page_offset, length);  in ceph_msg_data_next()
1030 		page = ceph_msg_data_pages_next(cursor, page_offset, length);  in ceph_msg_data_next()
1034 		page = ceph_msg_data_bio_next(cursor, page_offset, length);  in ceph_msg_data_next()
1038 		page = ceph_msg_data_bvecs_next(cursor, page_offset, length);  in ceph_msg_data_next()
1049 	BUG_ON(*length > cursor->resid);  in ceph_msg_data_next()
1058 void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor, size_t bytes)  in ceph_msg_data_advance()  argument
1062 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_advance()
1063 	switch (cursor->data->type) {  in ceph_msg_data_advance()
1065 		new_piece = ceph_msg_data_pagelist_advance(cursor, bytes);  in ceph_msg_data_advance()
1068 		new_piece = ceph_msg_data_pages_advance(cursor, bytes);  in ceph_msg_data_advance()
1072 		new_piece = ceph_msg_data_bio_advance(cursor, bytes);  in ceph_msg_data_advance()
1076 		new_piece = ceph_msg_data_bvecs_advance(cursor, bytes);  in ceph_msg_data_advance()
1083 	cursor->total_resid -= bytes;  in ceph_msg_data_advance()
1085 	if (!cursor->resid && cursor->total_resid) {  in ceph_msg_data_advance()
1086 		cursor->data++;  in ceph_msg_data_advance()
1087 		__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_advance()
1090 	cursor->need_crc = new_piece;  in ceph_msg_data_advance()