129 	struct xdr_buf *xdr = &rqst->rq_snd_buf;  in rpcrdma_args_inline()  local
133 	if (xdr->len > ep->re_max_inline_send)  in rpcrdma_args_inline()
136 	if (xdr->page_len) {  in rpcrdma_args_inline()
137 		remaining = xdr->page_len;  in rpcrdma_args_inline()
138 		offset = offset_in_page(xdr->page_base);  in rpcrdma_args_inline()
267 encode_rdma_segment(struct xdr_stream *xdr, struct rpcrdma_mr *mr)  in encode_rdma_segment()  argument
271 	p = xdr_reserve_space(xdr, 4 * sizeof(*p));  in encode_rdma_segment()
280 encode_read_segment(struct xdr_stream *xdr, struct rpcrdma_mr *mr,  in encode_read_segment()  argument
285 	p = xdr_reserve_space(xdr, 6 * sizeof(*p));  in encode_read_segment()
338 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_encode_read_list()  local
361 		if (encode_read_segment(xdr, mr, pos) < 0)  in rpcrdma_encode_read_list()
370 	if (xdr_stream_encode_item_absent(xdr) < 0)  in rpcrdma_encode_read_list()
395 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_encode_write_list()  local
412 	if (xdr_stream_encode_item_present(xdr) < 0)  in rpcrdma_encode_write_list()
414 	segcount = xdr_reserve_space(xdr, sizeof(*segcount));  in rpcrdma_encode_write_list()
425 		if (encode_rdma_segment(xdr, mr) < 0)  in rpcrdma_encode_write_list()
436 		if (encode_rdma_segment(xdr, ep->re_write_pad_mr) < 0)  in rpcrdma_encode_write_list()
451 	if (xdr_stream_encode_item_absent(xdr) < 0)  in rpcrdma_encode_write_list()
473 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_encode_reply_chunk()  local
480 		if (xdr_stream_encode_item_absent(xdr) < 0)  in rpcrdma_encode_reply_chunk()
490 	if (xdr_stream_encode_item_present(xdr) < 0)  in rpcrdma_encode_reply_chunk()
492 	segcount = xdr_reserve_space(xdr, sizeof(*segcount));  in rpcrdma_encode_reply_chunk()
503 		if (encode_rdma_segment(xdr, mr) < 0)  in rpcrdma_encode_reply_chunk()
597 				     struct xdr_buf *xdr)  in rpcrdma_prepare_pagelist()  argument
605 	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);  in rpcrdma_prepare_pagelist()
606 	page_base = offset_in_page(xdr->page_base);  in rpcrdma_prepare_pagelist()
607 	remaining = xdr->page_len;  in rpcrdma_prepare_pagelist()
637 				     struct xdr_buf *xdr,  in rpcrdma_prepare_tail_iov()  argument
643 	struct page *page = virt_to_page(xdr->tail[0].iov_base);  in rpcrdma_prepare_tail_iov()
664 				    struct xdr_buf *xdr)  in rpcrdma_pullup_tail_iov()  argument
668 	dst = (unsigned char *)xdr->head[0].iov_base;  in rpcrdma_pullup_tail_iov()
669 	dst += xdr->head[0].iov_len + xdr->page_len;  in rpcrdma_pullup_tail_iov()
670 	memmove(dst, xdr->tail[0].iov_base, xdr->tail[0].iov_len);  in rpcrdma_pullup_tail_iov()
671 	r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len;  in rpcrdma_pullup_tail_iov()
678 				    struct xdr_buf *xdr)  in rpcrdma_pullup_pagelist()  argument
684 	dst = (unsigned char *)xdr->head[0].iov_base;  in rpcrdma_pullup_pagelist()
685 	dst += xdr->head[0].iov_len;  in rpcrdma_pullup_pagelist()
686 	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);  in rpcrdma_pullup_pagelist()
687 	page_base = offset_in_page(xdr->page_base);  in rpcrdma_pullup_pagelist()
688 	remaining = xdr->page_len;  in rpcrdma_pullup_pagelist()
714 					struct xdr_buf *xdr)  in rpcrdma_prepare_noch_pullup()  argument
716 	if (unlikely(xdr->tail[0].iov_len))  in rpcrdma_prepare_noch_pullup()
717 		rpcrdma_pullup_tail_iov(r_xprt, req, xdr);  in rpcrdma_prepare_noch_pullup()
719 	if (unlikely(xdr->page_len))  in rpcrdma_prepare_noch_pullup()
720 		rpcrdma_pullup_pagelist(r_xprt, req, xdr);  in rpcrdma_prepare_noch_pullup()
723 	return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len);  in rpcrdma_prepare_noch_pullup()
728 					struct xdr_buf *xdr)  in rpcrdma_prepare_noch_mapped()  argument
730 	struct kvec *tail = &xdr->tail[0];  in rpcrdma_prepare_noch_mapped()
732 	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))  in rpcrdma_prepare_noch_mapped()
734 	if (xdr->page_len)  in rpcrdma_prepare_noch_mapped()
735 		if (!rpcrdma_prepare_pagelist(req, xdr))  in rpcrdma_prepare_noch_mapped()
738 		if (!rpcrdma_prepare_tail_iov(req, xdr,  in rpcrdma_prepare_noch_mapped()
750 				   struct xdr_buf *xdr)  in rpcrdma_prepare_readch()  argument
752 	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))  in rpcrdma_prepare_readch()
760 	if (xdr->tail[0].iov_len > 3) {  in rpcrdma_prepare_readch()
768 		page_base = offset_in_page(xdr->tail[0].iov_base);  in rpcrdma_prepare_readch()
769 		len = xdr->tail[0].iov_len;  in rpcrdma_prepare_readch()
772 		if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len))  in rpcrdma_prepare_readch()
792 				     struct xdr_buf *xdr,  in rpcrdma_prepare_send_sges()  argument
814 		if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr))  in rpcrdma_prepare_send_sges()
818 		if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr))  in rpcrdma_prepare_send_sges()
822 		if (!rpcrdma_prepare_readch(r_xprt, req, xdr))  in rpcrdma_prepare_send_sges()
863 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_marshal_req()  local
877 	xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf),  in rpcrdma_marshal_req()
882 	p = xdr_reserve_space(xdr, 4 * sizeof(*p));  in rpcrdma_marshal_req()
1125 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_is_bcall()  local
1132 	p = xdr_inline_decode(xdr, 0);  in rpcrdma_is_bcall()
1155 	p = xdr_inline_decode(xdr, 3 * sizeof(*p));  in rpcrdma_is_bcall()
1168 static int decode_rdma_segment(struct xdr_stream *xdr, u32 *length)  in decode_rdma_segment()  argument
1174 	p = xdr_inline_decode(xdr, 4 * sizeof(*p));  in decode_rdma_segment()
1183 static int decode_write_chunk(struct xdr_stream *xdr, u32 *length)  in decode_write_chunk()  argument
1188 	p = xdr_inline_decode(xdr, sizeof(*p));  in decode_write_chunk()
1195 		if (decode_rdma_segment(xdr, &seglength))  in decode_write_chunk()
1207 static int decode_read_list(struct xdr_stream *xdr)  in decode_read_list()  argument
1211 	p = xdr_inline_decode(xdr, sizeof(*p));  in decode_read_list()
1221 static int decode_write_list(struct xdr_stream *xdr, u32 *length)  in decode_write_list()  argument
1230 		p = xdr_inline_decode(xdr, sizeof(*p));  in decode_write_list()
1238 		if (decode_write_chunk(xdr, &chunklen))  in decode_write_list()
1246 static int decode_reply_chunk(struct xdr_stream *xdr, u32 *length)  in decode_reply_chunk()  argument
1250 	p = xdr_inline_decode(xdr, sizeof(*p));  in decode_reply_chunk()
1256 		if (decode_write_chunk(xdr, length))  in decode_reply_chunk()
1265 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_decode_msg()  local
1270 	if (decode_read_list(xdr))  in rpcrdma_decode_msg()
1272 	if (decode_write_list(xdr, &writelist))  in rpcrdma_decode_msg()
1274 	if (decode_reply_chunk(xdr, &replychunk))  in rpcrdma_decode_msg()
1282 	base = (char *)xdr_inline_decode(xdr, 0);  in rpcrdma_decode_msg()
1283 	rpclen = xdr_stream_remaining(xdr);  in rpcrdma_decode_msg()
1294 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_decode_nomsg()  local
1298 	if (decode_read_list(xdr))  in rpcrdma_decode_nomsg()
1300 	if (decode_write_list(xdr, &writelist))  in rpcrdma_decode_nomsg()
1302 	if (decode_reply_chunk(xdr, &replychunk))  in rpcrdma_decode_nomsg()
1320 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_decode_error()  local
1323 	p = xdr_inline_decode(xdr, sizeof(*p));  in rpcrdma_decode_error()
1329 		p = xdr_inline_decode(xdr, 2 * sizeof(*p));  in rpcrdma_decode_error()