109 static int ceph_tcp_recv(struct ceph_connection *con)  in ceph_tcp_recv()  argument
113 	dout("%s con %p %s %zu\n", __func__, con,  in ceph_tcp_recv()
114 	     iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",  in ceph_tcp_recv()
115 	     iov_iter_count(&con->v2.in_iter));  in ceph_tcp_recv()
116 	ret = do_recvmsg(con->sock, &con->v2.in_iter);  in ceph_tcp_recv()
117 	dout("%s con %p ret %d left %zu\n", __func__, con, ret,  in ceph_tcp_recv()
118 	     iov_iter_count(&con->v2.in_iter));  in ceph_tcp_recv()
196 static int ceph_tcp_send(struct ceph_connection *con)  in ceph_tcp_send()  argument
200 	dout("%s con %p have %zu try_sendpage %d\n", __func__, con,  in ceph_tcp_send()
201 	     iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);  in ceph_tcp_send()
202 	if (con->v2.out_iter_sendpage)  in ceph_tcp_send()
203 		ret = do_try_sendpage(con->sock, &con->v2.out_iter);  in ceph_tcp_send()
205 		ret = do_sendmsg(con->sock, &con->v2.out_iter);  in ceph_tcp_send()
206 	dout("%s con %p ret %d left %zu\n", __func__, con, ret,  in ceph_tcp_send()
207 	     iov_iter_count(&con->v2.out_iter));  in ceph_tcp_send()
211 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)  in add_in_kvec()  argument
213 	BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));  in add_in_kvec()
214 	WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));  in add_in_kvec()
216 	con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;  in add_in_kvec()
217 	con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;  in add_in_kvec()
218 	con->v2.in_kvec_cnt++;  in add_in_kvec()
220 	con->v2.in_iter.nr_segs++;  in add_in_kvec()
221 	con->v2.in_iter.count += len;  in add_in_kvec()
224 static void reset_in_kvecs(struct ceph_connection *con)  in reset_in_kvecs()  argument
226 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in reset_in_kvecs()
228 	con->v2.in_kvec_cnt = 0;  in reset_in_kvecs()
229 	iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);  in reset_in_kvecs()
232 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)  in set_in_bvec()  argument
234 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in set_in_bvec()
236 	con->v2.in_bvec = *bv;  in set_in_bvec()
237 	iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);  in set_in_bvec()
240 static void set_in_skip(struct ceph_connection *con, int len)  in set_in_skip()  argument
242 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in set_in_skip()
244 	dout("%s con %p len %d\n", __func__, con, len);  in set_in_skip()
245 	iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);  in set_in_skip()
248 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)  in add_out_kvec()  argument
250 	BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));  in add_out_kvec()
251 	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));  in add_out_kvec()
252 	WARN_ON(con->v2.out_zero);  in add_out_kvec()
254 	con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;  in add_out_kvec()
255 	con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;  in add_out_kvec()
256 	con->v2.out_kvec_cnt++;  in add_out_kvec()
258 	con->v2.out_iter.nr_segs++;  in add_out_kvec()
259 	con->v2.out_iter.count += len;  in add_out_kvec()
262 static void reset_out_kvecs(struct ceph_connection *con)  in reset_out_kvecs()  argument
264 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in reset_out_kvecs()
265 	WARN_ON(con->v2.out_zero);  in reset_out_kvecs()
267 	con->v2.out_kvec_cnt = 0;  in reset_out_kvecs()
269 	iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);  in reset_out_kvecs()
270 	con->v2.out_iter_sendpage = false;  in reset_out_kvecs()
273 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,  in set_out_bvec()  argument
276 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in set_out_bvec()
277 	WARN_ON(con->v2.out_zero);  in set_out_bvec()
279 	con->v2.out_bvec = *bv;  in set_out_bvec()
280 	con->v2.out_iter_sendpage = zerocopy;  in set_out_bvec()
281 	iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,  in set_out_bvec()
282 		      con->v2.out_bvec.bv_len);  in set_out_bvec()
285 static void set_out_bvec_zero(struct ceph_connection *con)  in set_out_bvec_zero()  argument
287 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in set_out_bvec_zero()
288 	WARN_ON(!con->v2.out_zero);  in set_out_bvec_zero()
290 	bvec_set_page(&con->v2.out_bvec, ceph_zero_page,  in set_out_bvec_zero()
291 		      min(con->v2.out_zero, (int)PAGE_SIZE), 0);  in set_out_bvec_zero()
292 	con->v2.out_iter_sendpage = true;  in set_out_bvec_zero()
293 	iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,  in set_out_bvec_zero()
294 		      con->v2.out_bvec.bv_len);  in set_out_bvec_zero()
297 static void out_zero_add(struct ceph_connection *con, int len)  in out_zero_add()  argument
299 	dout("%s con %p len %d\n", __func__, con, len);  in out_zero_add()
300 	con->v2.out_zero += len;  in out_zero_add()
303 static void *alloc_conn_buf(struct ceph_connection *con, int len)  in alloc_conn_buf()  argument
307 	dout("%s con %p len %d\n", __func__, con, len);  in alloc_conn_buf()
309 	if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))  in alloc_conn_buf()
316 	con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;  in alloc_conn_buf()
320 static void free_conn_bufs(struct ceph_connection *con)  in free_conn_bufs()  argument
322 	while (con->v2.conn_buf_cnt)  in free_conn_bufs()
323 		kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);  in free_conn_bufs()
326 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)  in add_in_sign_kvec()  argument
328 	BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));  in add_in_sign_kvec()
330 	con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;  in add_in_sign_kvec()
331 	con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;  in add_in_sign_kvec()
332 	con->v2.in_sign_kvec_cnt++;  in add_in_sign_kvec()
335 static void clear_in_sign_kvecs(struct ceph_connection *con)  in clear_in_sign_kvecs()  argument
337 	con->v2.in_sign_kvec_cnt = 0;  in clear_in_sign_kvecs()
340 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)  in add_out_sign_kvec()  argument
342 	BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));  in add_out_sign_kvec()
344 	con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;  in add_out_sign_kvec()
345 	con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;  in add_out_sign_kvec()
346 	con->v2.out_sign_kvec_cnt++;  in add_out_sign_kvec()
349 static void clear_out_sign_kvecs(struct ceph_connection *con)  in clear_out_sign_kvecs()  argument
351 	con->v2.out_sign_kvec_cnt = 0;  in clear_out_sign_kvecs()
354 static bool con_secure(struct ceph_connection *con)  in con_secure()  argument
356 	return con->v2.con_mode == CEPH_CON_MODE_SECURE;  in con_secure()
565 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)  in encode_epilogue_plain()  argument
567 	con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :  in encode_epilogue_plain()
569 	cpu_to_le32s(&con->v2.out_epil.front_crc);  in encode_epilogue_plain()
570 	cpu_to_le32s(&con->v2.out_epil.middle_crc);  in encode_epilogue_plain()
571 	cpu_to_le32s(&con->v2.out_epil.data_crc);  in encode_epilogue_plain()
574 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)  in encode_epilogue_secure()  argument
576 	memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));  in encode_epilogue_secure()
577 	con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :  in encode_epilogue_secure()
639 static int verify_control_crc(struct ceph_connection *con)  in verify_control_crc()  argument
641 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in verify_control_crc()
644 	WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);  in verify_control_crc()
645 	WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);  in verify_control_crc()
647 	crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);  in verify_control_crc()
648 	expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);  in verify_control_crc()
658 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,  in verify_epilogue_crcs()  argument
661 	if (front_len(con->in_msg)) {  in verify_epilogue_crcs()
662 		con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,  in verify_epilogue_crcs()
663 					   front_len(con->in_msg));  in verify_epilogue_crcs()
665 		WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));  in verify_epilogue_crcs()
666 		con->in_front_crc = -1;  in verify_epilogue_crcs()
669 	if (middle_len(con->in_msg))  in verify_epilogue_crcs()
670 		con->in_middle_crc = crc32c(-1,  in verify_epilogue_crcs()
671 					    con->in_msg->middle->vec.iov_base,  in verify_epilogue_crcs()
672 					    middle_len(con->in_msg));  in verify_epilogue_crcs()
673 	else if (data_len(con->in_msg))  in verify_epilogue_crcs()
674 		con->in_middle_crc = -1;  in verify_epilogue_crcs()
676 		con->in_middle_crc = 0;  in verify_epilogue_crcs()
678 	if (!data_len(con->in_msg))  in verify_epilogue_crcs()
679 		con->in_data_crc = 0;  in verify_epilogue_crcs()
681 	dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,  in verify_epilogue_crcs()
682 	     con->in_front_crc, con->in_middle_crc, con->in_data_crc);  in verify_epilogue_crcs()
684 	if (con->in_front_crc != front_crc) {  in verify_epilogue_crcs()
686 		       con->in_front_crc, front_crc);  in verify_epilogue_crcs()
689 	if (con->in_middle_crc != middle_crc) {  in verify_epilogue_crcs()
691 		       con->in_middle_crc, middle_crc);  in verify_epilogue_crcs()
694 	if (con->in_data_crc != data_crc) {  in verify_epilogue_crcs()
696 		       con->in_data_crc, data_crc);  in verify_epilogue_crcs()
703 static int setup_crypto(struct ceph_connection *con,  in setup_crypto()  argument
711 	     __func__, con, con->v2.con_mode, session_key_len, con_secret_len);  in setup_crypto()
712 	WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);  in setup_crypto()
714 	if (con->v2.con_mode != CEPH_CON_MODE_CRC &&  in setup_crypto()
715 	    con->v2.con_mode != CEPH_CON_MODE_SECURE) {  in setup_crypto()
716 		pr_err("bad con_mode %d\n", con->v2.con_mode);  in setup_crypto()
721 		WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);  in setup_crypto()
727 	con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);  in setup_crypto()
729 	if (IS_ERR(con->v2.hmac_tfm)) {  in setup_crypto()
730 		ret = PTR_ERR(con->v2.hmac_tfm);  in setup_crypto()
731 		con->v2.hmac_tfm = NULL;  in setup_crypto()
737 		crypto_shash_alignmask(con->v2.hmac_tfm));  in setup_crypto()
738 	ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,  in setup_crypto()
745 	if (con->v2.con_mode == CEPH_CON_MODE_CRC) {  in setup_crypto()
756 	con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);  in setup_crypto()
758 	if (IS_ERR(con->v2.gcm_tfm)) {  in setup_crypto()
759 		ret = PTR_ERR(con->v2.gcm_tfm);  in setup_crypto()
760 		con->v2.gcm_tfm = NULL;  in setup_crypto()
766 		crypto_aead_alignmask(con->v2.gcm_tfm));  in setup_crypto()
767 	ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);  in setup_crypto()
773 	WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);  in setup_crypto()
774 	ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);  in setup_crypto()
780 	con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);  in setup_crypto()
781 	if (!con->v2.gcm_req) {  in setup_crypto()
786 	crypto_init_wait(&con->v2.gcm_wait);  in setup_crypto()
787 	aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in setup_crypto()
788 				  crypto_req_done, &con->v2.gcm_wait);  in setup_crypto()
790 	memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,  in setup_crypto()
792 	memcpy(&con->v2.out_gcm_nonce,  in setup_crypto()
798 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,  in hmac_sha256()  argument
801 	SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm);  /* tfm arg is ignored */  in hmac_sha256()
805 	dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,  in hmac_sha256()
806 	     con->v2.hmac_tfm, kvec_cnt);  in hmac_sha256()
808 	if (!con->v2.hmac_tfm) {  in hmac_sha256()
813 	desc->tfm = con->v2.hmac_tfm;  in hmac_sha256()
820 			crypto_shash_alignmask(con->v2.hmac_tfm));  in hmac_sha256()
842 static int gcm_crypt(struct ceph_connection *con, bool encrypt,  in gcm_crypt()  argument
849 	nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;  in gcm_crypt()
851 	aead_request_set_ad(con->v2.gcm_req, 0);  /* no AAD */  in gcm_crypt()
852 	aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);  in gcm_crypt()
853 	ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :  in gcm_crypt()
854 					crypto_aead_decrypt(con->v2.gcm_req),  in gcm_crypt()
855 			      &con->v2.gcm_wait);  in gcm_crypt()
1064 static int decrypt_preamble(struct ceph_connection *con)  in decrypt_preamble()  argument
1068 	sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);  in decrypt_preamble()
1069 	return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);  in decrypt_preamble()
1072 static int decrypt_control_remainder(struct ceph_connection *con)  in decrypt_control_remainder()  argument
1074 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in decrypt_control_remainder()
1079 	WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);  in decrypt_control_remainder()
1080 	WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);  in decrypt_control_remainder()
1083 	sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);  in decrypt_control_remainder()
1084 	sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);  in decrypt_control_remainder()
1086 	return gcm_crypt(con, false, sgs, sgs,  in decrypt_control_remainder()
1091 static int process_v2_sparse_read(struct ceph_connection *con,  in process_v2_sparse_read()  argument
1094 	struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;  in process_v2_sparse_read()
1100 		ret = con->ops->sparse_read(con, cursor, &buf);  in process_v2_sparse_read()
1109 			struct page *spage = con->v2.in_enc_pages[idx];  in process_v2_sparse_read()
1130 static int decrypt_tail(struct ceph_connection *con)  in decrypt_tail()  argument
1135 	bool sparse = !!con->in_msg->sparse_read_total;  in decrypt_tail()
1140 	tail_len = tail_onwire_len(con->in_msg, true);  in decrypt_tail()
1141 	ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,  in decrypt_tail()
1142 					con->v2.in_enc_page_cnt, 0, tail_len,  in decrypt_tail()
1148 		dpos = padded_len(front_len(con->in_msg) + padded_len(middle_len(con->in_msg)));  in decrypt_tail()
1149 		pages = con->v2.in_enc_pages;  in decrypt_tail()
1152 	ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),  in decrypt_tail()
1153 				MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),  in decrypt_tail()
1154 				con->v2.in_buf, pages, dpos, true);  in decrypt_tail()
1158 	dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,  in decrypt_tail()
1159 	     con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);  in decrypt_tail()
1160 	ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);  in decrypt_tail()
1164 	if (sparse && data_len(con->in_msg)) {  in decrypt_tail()
1165 		ret = process_v2_sparse_read(con, con->v2.in_enc_pages, dpos);  in decrypt_tail()
1170 	WARN_ON(!con->v2.in_enc_page_cnt);  in decrypt_tail()
1171 	ceph_release_page_vector(con->v2.in_enc_pages,  in decrypt_tail()
1172 				 con->v2.in_enc_page_cnt);  in decrypt_tail()
1173 	con->v2.in_enc_pages = NULL;  in decrypt_tail()
1174 	con->v2.in_enc_page_cnt = 0;  in decrypt_tail()
1182 static int prepare_banner(struct ceph_connection *con)  in prepare_banner()  argument
1187 	buf = alloc_conn_buf(con, buf_len);  in prepare_banner()
1198 	add_out_kvec(con, buf, buf_len);  in prepare_banner()
1199 	add_out_sign_kvec(con, buf, buf_len);  in prepare_banner()
1200 	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);  in prepare_banner()
1221 static void prepare_head_plain(struct ceph_connection *con, void *base,  in prepare_head_plain()  argument
1235 		add_out_kvec(con, base, base_len);  in prepare_head_plain()
1237 			add_out_sign_kvec(con, base, base_len);  in prepare_head_plain()
1241 	add_out_kvec(con, base, crcp - base);  in prepare_head_plain()
1242 	add_out_kvec(con, extdata, extdata_len);  in prepare_head_plain()
1243 	add_out_kvec(con, crcp, CEPH_CRC_LEN);  in prepare_head_plain()
1245 		add_out_sign_kvec(con, base, crcp - base);  in prepare_head_plain()
1246 		add_out_sign_kvec(con, extdata, extdata_len);  in prepare_head_plain()
1247 		add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);  in prepare_head_plain()
1251 static int prepare_head_secure_small(struct ceph_connection *con,  in prepare_head_secure_small()  argument
1263 	ret = gcm_crypt(con, true, &sg, &sg,  in prepare_head_secure_small()
1268 	add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);  in prepare_head_secure_small()
1292 static int prepare_head_secure_big(struct ceph_connection *con,  in prepare_head_secure_big()  argument
1305 	ret = gcm_crypt(con, true, sgs, sgs, rem - base);  in prepare_head_secure_big()
1314 	ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);  in prepare_head_secure_big()
1318 	add_out_kvec(con, base, rem - base);  in prepare_head_secure_big()
1319 	add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);  in prepare_head_secure_big()
1320 	add_out_kvec(con, rem, pmbl_tag - rem);  in prepare_head_secure_big()
1324 static int __prepare_control(struct ceph_connection *con, int tag,  in __prepare_control()  argument
1332 	dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,  in __prepare_control()
1342 	if (con_secure(con)) {  in __prepare_control()
1348 			ret = prepare_head_secure_small(con, base, ctrl_len);  in __prepare_control()
1351 			ret = prepare_head_secure_big(con, base, ctrl_len);  in __prepare_control()
1355 		prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,  in __prepare_control()
1359 	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);  in __prepare_control()
1363 static int prepare_control(struct ceph_connection *con, int tag,  in prepare_control()  argument
1366 	return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);  in prepare_control()
1369 static int prepare_hello(struct ceph_connection *con)  in prepare_hello()  argument
1374 	ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);  in prepare_hello()
1375 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));  in prepare_hello()
1381 	ceph_encode_entity_addr(&p, &con->peer_addr);  in prepare_hello()
1384 	return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,  in prepare_hello()
1391 static int prepare_auth_request(struct ceph_connection *con)  in prepare_auth_request()  argument
1399 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));  in prepare_auth_request()
1403 	mutex_unlock(&con->mutex);  in prepare_auth_request()
1404 	ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,  in prepare_auth_request()
1406 	mutex_lock(&con->mutex);  in prepare_auth_request()
1407 	if (con->state != CEPH_CON_S_V2_HELLO) {  in prepare_auth_request()
1408 		dout("%s con %p state changed to %d\n", __func__, con,  in prepare_auth_request()
1409 		     con->state);  in prepare_auth_request()
1413 	dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);  in prepare_auth_request()
1417 	authorizer_copy = alloc_conn_buf(con, authorizer_len);  in prepare_auth_request()
1423 	return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,  in prepare_auth_request()
1427 static int prepare_auth_request_more(struct ceph_connection *con,  in prepare_auth_request_more()  argument
1436 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));  in prepare_auth_request_more()
1440 	mutex_unlock(&con->mutex);  in prepare_auth_request_more()
1441 	ret = con->ops->handle_auth_reply_more(con, reply, reply_len,  in prepare_auth_request_more()
1444 	mutex_lock(&con->mutex);  in prepare_auth_request_more()
1445 	if (con->state != CEPH_CON_S_V2_AUTH) {  in prepare_auth_request_more()
1446 		dout("%s con %p state changed to %d\n", __func__, con,  in prepare_auth_request_more()
1447 		     con->state);  in prepare_auth_request_more()
1451 	dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);  in prepare_auth_request_more()
1455 	return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,  in prepare_auth_request_more()
1459 static int prepare_auth_signature(struct ceph_connection *con)  in prepare_auth_signature()  argument
1464 	buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,  in prepare_auth_signature()
1465 						  con_secure(con)));  in prepare_auth_signature()
1469 	ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,  in prepare_auth_signature()
1474 	return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,  in prepare_auth_signature()
1478 static int prepare_client_ident(struct ceph_connection *con)  in prepare_client_ident()  argument
1480 	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;  in prepare_client_ident()
1481 	struct ceph_client *client = from_msgr(con->msgr);  in prepare_client_ident()
1486 	WARN_ON(con->v2.server_cookie);  in prepare_client_ident()
1487 	WARN_ON(con->v2.connect_seq);  in prepare_client_ident()
1488 	WARN_ON(con->v2.peer_global_seq);  in prepare_client_ident()
1490 	if (!con->v2.client_cookie) {  in prepare_client_ident()
1492 			get_random_bytes(&con->v2.client_cookie,  in prepare_client_ident()
1493 					 sizeof(con->v2.client_cookie));  in prepare_client_ident()
1494 		} while (!con->v2.client_cookie);  in prepare_client_ident()
1495 		dout("%s con %p generated cookie 0x%llx\n", __func__, con,  in prepare_client_ident()
1496 		     con->v2.client_cookie);  in prepare_client_ident()
1498 		dout("%s con %p cookie already set 0x%llx\n", __func__, con,  in prepare_client_ident()
1499 		     con->v2.client_cookie);  in prepare_client_ident()
1503 	     __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),  in prepare_client_ident()
1504 	     ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),  in prepare_client_ident()
1505 	     global_id, con->v2.global_seq, client->supported_features,  in prepare_client_ident()
1506 	     client->required_features, con->v2.client_cookie);  in prepare_client_ident()
1509 		   ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;  in prepare_client_ident()
1510 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));  in prepare_client_ident()
1518 	ceph_encode_entity_addr(&p, &con->peer_addr);  in prepare_client_ident()
1520 	ceph_encode_64(&p, con->v2.global_seq);  in prepare_client_ident()
1524 	ceph_encode_64(&p, con->v2.client_cookie);  in prepare_client_ident()
1527 	return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);  in prepare_client_ident()
1530 static int prepare_session_reconnect(struct ceph_connection *con)  in prepare_session_reconnect()  argument
1532 	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;  in prepare_session_reconnect()
1536 	WARN_ON(!con->v2.client_cookie);  in prepare_session_reconnect()
1537 	WARN_ON(!con->v2.server_cookie);  in prepare_session_reconnect()
1538 	WARN_ON(!con->v2.connect_seq);  in prepare_session_reconnect()
1539 	WARN_ON(!con->v2.peer_global_seq);  in prepare_session_reconnect()
1542 	     __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),  in prepare_session_reconnect()
1543 	     con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,  in prepare_session_reconnect()
1544 	     con->v2.connect_seq, con->in_seq);  in prepare_session_reconnect()
1547 	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));  in prepare_session_reconnect()
1555 	ceph_encode_64(&p, con->v2.client_cookie);  in prepare_session_reconnect()
1556 	ceph_encode_64(&p, con->v2.server_cookie);  in prepare_session_reconnect()
1557 	ceph_encode_64(&p, con->v2.global_seq);  in prepare_session_reconnect()
1558 	ceph_encode_64(&p, con->v2.connect_seq);  in prepare_session_reconnect()
1559 	ceph_encode_64(&p, con->in_seq);  in prepare_session_reconnect()
1562 	return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);  in prepare_session_reconnect()
1565 static int prepare_keepalive2(struct ceph_connection *con)  in prepare_keepalive2()  argument
1567 	struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);  in prepare_keepalive2()
1571 	dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,  in prepare_keepalive2()
1576 	reset_out_kvecs(con);  in prepare_keepalive2()
1577 	return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,  in prepare_keepalive2()
1581 static int prepare_ack(struct ceph_connection *con)  in prepare_ack()  argument
1585 	dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,  in prepare_ack()
1586 	     con->in_seq_acked, con->in_seq);  in prepare_ack()
1587 	con->in_seq_acked = con->in_seq;  in prepare_ack()
1589 	p = CTRL_BODY(con->v2.out_buf);  in prepare_ack()
1590 	ceph_encode_64(&p, con->in_seq_acked);  in prepare_ack()
1592 	reset_out_kvecs(con);  in prepare_ack()
1593 	return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);  in prepare_ack()
1596 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)  in prepare_epilogue_plain()  argument
1598 	dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,  in prepare_epilogue_plain()
1599 	     con->out_msg, aborted, con->v2.out_epil.front_crc,  in prepare_epilogue_plain()
1600 	     con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);  in prepare_epilogue_plain()
1602 	encode_epilogue_plain(con, aborted);  in prepare_epilogue_plain()
1603 	add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_epilogue_plain()
1610 static void prepare_message_plain(struct ceph_connection *con)  in prepare_message_plain()  argument
1612 	struct ceph_msg *msg = con->out_msg;  in prepare_message_plain()
1614 	prepare_head_plain(con, con->v2.out_buf,  in prepare_message_plain()
1623 			con->v2.out_state = OUT_S_FINISH_MESSAGE;  in prepare_message_plain()
1627 		con->v2.out_epil.front_crc = -1;  in prepare_message_plain()
1628 		con->v2.out_epil.middle_crc = -1;  in prepare_message_plain()
1629 		con->v2.out_state = OUT_S_QUEUE_DATA;  in prepare_message_plain()
1634 		con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,  in prepare_message_plain()
1636 		add_out_kvec(con, msg->front.iov_base, front_len(msg));  in prepare_message_plain()
1639 		con->v2.out_epil.front_crc = -1;  in prepare_message_plain()
1643 		con->v2.out_epil.middle_crc =  in prepare_message_plain()
1645 		add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));  in prepare_message_plain()
1647 		con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;  in prepare_message_plain()
1651 		con->v2.out_state = OUT_S_QUEUE_DATA;  in prepare_message_plain()
1653 		con->v2.out_epil.data_crc = 0;  in prepare_message_plain()
1654 		prepare_epilogue_plain(con, false);  in prepare_message_plain()
1655 		con->v2.out_state = OUT_S_FINISH_MESSAGE;  in prepare_message_plain()
1666 static int prepare_message_secure(struct ceph_connection *con)  in prepare_message_secure()  argument
1676 	ret = prepare_head_secure_small(con, con->v2.out_buf,  in prepare_message_secure()
1681 	tail_len = tail_onwire_len(con->out_msg, true);  in prepare_message_secure()
1687 		con->v2.out_state = OUT_S_FINISH_MESSAGE;  in prepare_message_secure()
1691 	encode_epilogue_secure(con, false);  in prepare_message_secure()
1692 	ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,  in prepare_message_secure()
1693 				&con->v2.out_epil, NULL, 0, false);  in prepare_message_secure()
1704 	WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);  in prepare_message_secure()
1705 	con->v2.out_enc_pages = enc_pages;  in prepare_message_secure()
1706 	con->v2.out_enc_page_cnt = enc_page_cnt;  in prepare_message_secure()
1707 	con->v2.out_enc_resid = tail_len;  in prepare_message_secure()
1708 	con->v2.out_enc_i = 0;  in prepare_message_secure()
1715 	ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,  in prepare_message_secure()
1720 	dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,  in prepare_message_secure()
1721 	     con->out_msg, sgt.orig_nents, enc_page_cnt);  in prepare_message_secure()
1722 	con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;  in prepare_message_secure()
1730 static int prepare_message(struct ceph_connection *con)  in prepare_message()  argument
1734 		front_len(con->out_msg),  in prepare_message()
1735 		middle_len(con->out_msg),  in prepare_message()
1736 		data_len(con->out_msg)  in prepare_message()
1741 	dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,  in prepare_message()
1742 	     con->out_msg, lens[0], lens[1], lens[2], lens[3]);  in prepare_message()
1744 	if (con->in_seq > con->in_seq_acked) {  in prepare_message()
1745 		dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,  in prepare_message()
1746 		     con->in_seq_acked, con->in_seq);  in prepare_message()
1747 		con->in_seq_acked = con->in_seq;  in prepare_message()
1750 	reset_out_kvecs(con);  in prepare_message()
1752 	encode_preamble(&desc, con->v2.out_buf);  in prepare_message()
1753 	fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,  in prepare_message()
1754 		     con->in_seq_acked);  in prepare_message()
1756 	if (con_secure(con)) {  in prepare_message()
1757 		ret = prepare_message_secure(con);  in prepare_message()
1761 		prepare_message_plain(con);  in prepare_message()
1764 	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);  in prepare_message()
1768 static int prepare_read_banner_prefix(struct ceph_connection *con)  in prepare_read_banner_prefix()  argument
1772 	buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);  in prepare_read_banner_prefix()
1776 	reset_in_kvecs(con);  in prepare_read_banner_prefix()
1777 	add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);  in prepare_read_banner_prefix()
1778 	add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);  in prepare_read_banner_prefix()
1779 	con->state = CEPH_CON_S_V2_BANNER_PREFIX;  in prepare_read_banner_prefix()
1783 static int prepare_read_banner_payload(struct ceph_connection *con,  in prepare_read_banner_payload()  argument
1788 	buf = alloc_conn_buf(con, payload_len);  in prepare_read_banner_payload()
1792 	reset_in_kvecs(con);  in prepare_read_banner_payload()
1793 	add_in_kvec(con, buf, payload_len);  in prepare_read_banner_payload()
1794 	add_in_sign_kvec(con, buf, payload_len);  in prepare_read_banner_payload()
1795 	con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;  in prepare_read_banner_payload()
1799 static void prepare_read_preamble(struct ceph_connection *con)  in prepare_read_preamble()  argument
1801 	reset_in_kvecs(con);  in prepare_read_preamble()
1802 	add_in_kvec(con, con->v2.in_buf,  in prepare_read_preamble()
1803 		    con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :  in prepare_read_preamble()
1805 	con->v2.in_state = IN_S_HANDLE_PREAMBLE;  in prepare_read_preamble()
1808 static int prepare_read_control(struct ceph_connection *con)  in prepare_read_control()  argument
1810 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in prepare_read_control()
1814 	reset_in_kvecs(con);  in prepare_read_control()
1815 	if (con->state == CEPH_CON_S_V2_HELLO ||  in prepare_read_control()
1816 	    con->state == CEPH_CON_S_V2_AUTH) {  in prepare_read_control()
1818 		buf = alloc_conn_buf(con, head_len);  in prepare_read_control()
1823 		memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);  in prepare_read_control()
1825 		add_in_kvec(con, CTRL_BODY(buf), ctrl_len);  in prepare_read_control()
1826 		add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);  in prepare_read_control()
1827 		add_in_sign_kvec(con, buf, head_len);  in prepare_read_control()
1830 			buf = alloc_conn_buf(con, ctrl_len);  in prepare_read_control()
1834 			add_in_kvec(con, buf, ctrl_len);  in prepare_read_control()
1836 			add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);  in prepare_read_control()
1838 		add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);  in prepare_read_control()
1840 	con->v2.in_state = IN_S_HANDLE_CONTROL;  in prepare_read_control()
1844 static int prepare_read_control_remainder(struct ceph_connection *con)  in prepare_read_control_remainder()  argument
1846 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in prepare_read_control_remainder()
1850 	buf = alloc_conn_buf(con, ctrl_len);  in prepare_read_control_remainder()
1854 	memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);  in prepare_read_control_remainder()
1856 	reset_in_kvecs(con);  in prepare_read_control_remainder()
1857 	add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);  in prepare_read_control_remainder()
1858 	add_in_kvec(con, con->v2.in_buf,  in prepare_read_control_remainder()
1860 	con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;  in prepare_read_control_remainder()
1864 static int prepare_read_data(struct ceph_connection *con)  in prepare_read_data()  argument
1868 	con->in_data_crc = -1;  in prepare_read_data()
1869 	ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,  in prepare_read_data()
1870 				  data_len(con->in_msg));  in prepare_read_data()
1872 	get_bvec_at(&con->v2.in_cursor, &bv);  in prepare_read_data()
1873 	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_read_data()
1874 		if (unlikely(!con->bounce_page)) {  in prepare_read_data()
1875 			con->bounce_page = alloc_page(GFP_NOIO);  in prepare_read_data()
1876 			if (!con->bounce_page) {  in prepare_read_data()
1882 		bv.bv_page = con->bounce_page;  in prepare_read_data()
1885 	set_in_bvec(con, &bv);  in prepare_read_data()
1886 	con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;  in prepare_read_data()
1890 static void prepare_read_data_cont(struct ceph_connection *con)  in prepare_read_data_cont()  argument
1894 	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_read_data_cont()
1895 		con->in_data_crc = crc32c(con->in_data_crc,  in prepare_read_data_cont()
1896 					  page_address(con->bounce_page),  in prepare_read_data_cont()
1897 					  con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1899 		get_bvec_at(&con->v2.in_cursor, &bv);  in prepare_read_data_cont()
1901 			       page_address(con->bounce_page),  in prepare_read_data_cont()
1902 			       con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1904 		con->in_data_crc = ceph_crc32c_page(con->in_data_crc,  in prepare_read_data_cont()
1905 						    con->v2.in_bvec.bv_page,  in prepare_read_data_cont()
1906 						    con->v2.in_bvec.bv_offset,  in prepare_read_data_cont()
1907 						    con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1910 	ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);  in prepare_read_data_cont()
1911 	if (con->v2.in_cursor.total_resid) {  in prepare_read_data_cont()
1912 		get_bvec_at(&con->v2.in_cursor, &bv);  in prepare_read_data_cont()
1913 		if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_read_data_cont()
1914 			bv.bv_page = con->bounce_page;  in prepare_read_data_cont()
1917 		set_in_bvec(con, &bv);  in prepare_read_data_cont()
1918 		WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);  in prepare_read_data_cont()
1925 	reset_in_kvecs(con);  in prepare_read_data_cont()
1926 	add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_read_data_cont()
1927 	con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_read_data_cont()
1930 static int prepare_sparse_read_cont(struct ceph_connection *con)  in prepare_sparse_read_cont()  argument
1935 	struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;  in prepare_sparse_read_cont()
1937 	WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT);  in prepare_sparse_read_cont()
1939 	if (iov_iter_is_bvec(&con->v2.in_iter)) {  in prepare_sparse_read_cont()
1940 		if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_sparse_read_cont()
1941 			con->in_data_crc = crc32c(con->in_data_crc,  in prepare_sparse_read_cont()
1942 						  page_address(con->bounce_page),  in prepare_sparse_read_cont()
1943 						  con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1946 				       page_address(con->bounce_page),  in prepare_sparse_read_cont()
1947 				       con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1949 			con->in_data_crc = ceph_crc32c_page(con->in_data_crc,  in prepare_sparse_read_cont()
1950 							    con->v2.in_bvec.bv_page,  in prepare_sparse_read_cont()
1951 							    con->v2.in_bvec.bv_offset,  in prepare_sparse_read_cont()
1952 							    con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1955 		ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len);  in prepare_sparse_read_cont()
1956 		cursor->sr_resid -= con->v2.in_bvec.bv_len;  in prepare_sparse_read_cont()
1958 		     con->v2.in_bvec.bv_len, cursor->sr_resid);  in prepare_sparse_read_cont()
1964 			if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_sparse_read_cont()
1965 				bv.bv_page = con->bounce_page;  in prepare_sparse_read_cont()
1968 			set_in_bvec(con, &bv);  in prepare_sparse_read_cont()
1969 			con->v2.data_len_remain -= bv.bv_len;  in prepare_sparse_read_cont()
1972 	} else if (iov_iter_is_kvec(&con->v2.in_iter)) {  in prepare_sparse_read_cont()
1974 		if (con->v2.in_kvec_cnt) {  in prepare_sparse_read_cont()
1975 			WARN_ON_ONCE(con->v2.in_kvec_cnt > 1);  in prepare_sparse_read_cont()
1976 			con->in_data_crc = crc32c(con->in_data_crc,  in prepare_sparse_read_cont()
1977 						  con->v2.in_kvecs[0].iov_base,  in prepare_sparse_read_cont()
1978 						  con->v2.in_kvecs[0].iov_len);  in prepare_sparse_read_cont()
1985 	ret = con->ops->sparse_read(con, cursor, &buf);  in prepare_sparse_read_cont()
1990 		reset_in_kvecs(con);  in prepare_sparse_read_cont()
1991 		add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_sparse_read_cont()
1992 		con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_sparse_read_cont()
1998 		reset_in_kvecs(con);  in prepare_sparse_read_cont()
1999 		add_in_kvec(con, buf, ret);  in prepare_sparse_read_cont()
2000 		con->v2.data_len_remain -= ret;  in prepare_sparse_read_cont()
2012 	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {  in prepare_sparse_read_cont()
2013 		if (unlikely(!con->bounce_page)) {  in prepare_sparse_read_cont()
2014 			con->bounce_page = alloc_page(GFP_NOIO);  in prepare_sparse_read_cont()
2015 			if (!con->bounce_page) {  in prepare_sparse_read_cont()
2021 		bv.bv_page = con->bounce_page;  in prepare_sparse_read_cont()
2024 	set_in_bvec(con, &bv);  in prepare_sparse_read_cont()
2025 	con->v2.data_len_remain -= ret;  in prepare_sparse_read_cont()
2029 static int prepare_sparse_read_data(struct ceph_connection *con)  in prepare_sparse_read_data()  argument
2031 	struct ceph_msg *msg = con->in_msg;  in prepare_sparse_read_data()
2035 	if (WARN_ON_ONCE(!con->ops->sparse_read))  in prepare_sparse_read_data()
2038 	if (!con_secure(con))  in prepare_sparse_read_data()
2039 		con->in_data_crc = -1;  in prepare_sparse_read_data()
2041 	reset_in_kvecs(con);  in prepare_sparse_read_data()
2042 	con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT;  in prepare_sparse_read_data()
2043 	con->v2.data_len_remain = data_len(msg);  in prepare_sparse_read_data()
2044 	return prepare_sparse_read_cont(con);  in prepare_sparse_read_data()
2047 static int prepare_read_tail_plain(struct ceph_connection *con)  in prepare_read_tail_plain()  argument
2049 	struct ceph_msg *msg = con->in_msg;  in prepare_read_tail_plain()
2053 		return prepare_read_data(con);  in prepare_read_tail_plain()
2056 	reset_in_kvecs(con);  in prepare_read_tail_plain()
2058 		add_in_kvec(con, msg->front.iov_base, front_len(msg));  in prepare_read_tail_plain()
2062 		add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));  in prepare_read_tail_plain()
2068 			con->v2.in_state = IN_S_PREPARE_SPARSE_DATA;  in prepare_read_tail_plain()
2070 			con->v2.in_state = IN_S_PREPARE_READ_DATA;  in prepare_read_tail_plain()
2072 		add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);  in prepare_read_tail_plain()
2073 		con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_read_tail_plain()
2078 static void prepare_read_enc_page(struct ceph_connection *con)  in prepare_read_enc_page()  argument
2082 	dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,  in prepare_read_enc_page()
2083 	     con->v2.in_enc_resid);  in prepare_read_enc_page()
2084 	WARN_ON(!con->v2.in_enc_resid);  in prepare_read_enc_page()
2086 	bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i],  in prepare_read_enc_page()
2087 		      min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0);  in prepare_read_enc_page()
2089 	set_in_bvec(con, &bv);  in prepare_read_enc_page()
2090 	con->v2.in_enc_i++;  in prepare_read_enc_page()
2091 	con->v2.in_enc_resid -= bv.bv_len;  in prepare_read_enc_page()
2093 	if (con->v2.in_enc_resid) {  in prepare_read_enc_page()
2094 		con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;  in prepare_read_enc_page()
2102 	WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);  in prepare_read_enc_page()
2103 	con->v2.in_state = IN_S_HANDLE_EPILOGUE;  in prepare_read_enc_page()
2106 static int prepare_read_tail_secure(struct ceph_connection *con)  in prepare_read_tail_secure()  argument
2112 	tail_len = tail_onwire_len(con->in_msg, true);  in prepare_read_tail_secure()
2120 	WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);  in prepare_read_tail_secure()
2121 	con->v2.in_enc_pages = enc_pages;  in prepare_read_tail_secure()
2122 	con->v2.in_enc_page_cnt = enc_page_cnt;  in prepare_read_tail_secure()
2123 	con->v2.in_enc_resid = tail_len;  in prepare_read_tail_secure()
2124 	con->v2.in_enc_i = 0;  in prepare_read_tail_secure()
2126 	prepare_read_enc_page(con);  in prepare_read_tail_secure()
2130 static void __finish_skip(struct ceph_connection *con)  in __finish_skip()  argument
2132 	con->in_seq++;  in __finish_skip()
2133 	prepare_read_preamble(con);  in __finish_skip()
2136 static void prepare_skip_message(struct ceph_connection *con)  in prepare_skip_message()  argument
2138 	struct ceph_frame_desc *desc = &con->v2.in_desc;  in prepare_skip_message()
2141 	dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],  in prepare_skip_message()
2145 				     desc->fd_lens[3], con_secure(con));  in prepare_skip_message()
2147 		__finish_skip(con);  in prepare_skip_message()
2149 		set_in_skip(con, tail_len);  in prepare_skip_message()
2150 		con->v2.in_state = IN_S_FINISH_SKIP;  in prepare_skip_message()
2154 static int process_banner_prefix(struct ceph_connection *con)  in process_banner_prefix()  argument
2159 	WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);  in process_banner_prefix()
2161 	p = con->v2.in_kvecs[0].iov_base;  in process_banner_prefix()
2164 			con->error_msg = "server is speaking msgr1 protocol";  in process_banner_prefix()
2166 			con->error_msg = "protocol error, bad banner";  in process_banner_prefix()
2172 	dout("%s con %p payload_len %d\n", __func__, con, payload_len);  in process_banner_prefix()
2174 	return prepare_read_banner_payload(con, payload_len);  in process_banner_prefix()
2177 static int process_banner_payload(struct ceph_connection *con)  in process_banner_payload()  argument
2179 	void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;  in process_banner_payload()
2186 	p = con->v2.in_kvecs[0].iov_base;  in process_banner_payload()
2191 	     __func__, con, server_feat, server_req_feat);  in process_banner_payload()
2196 		con->error_msg = "missing required protocol features";  in process_banner_payload()
2202 		con->error_msg = "missing required protocol features";  in process_banner_payload()
2207 	ret = prepare_hello(con);  in process_banner_payload()
2213 	con->state = CEPH_CON_S_V2_HELLO;  in process_banner_payload()
2214 	prepare_read_preamble(con);  in process_banner_payload()
2222 static int process_hello(struct ceph_connection *con, void *p, void *end)  in process_hello()  argument
2224 	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;  in process_hello()
2229 	if (con->state != CEPH_CON_S_V2_HELLO) {  in process_hello()
2230 		con->error_msg = "protocol error, unexpected hello";  in process_hello()
2241 	dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,  in process_hello()
2244 	if (entity_type != con->peer_name.type) {  in process_hello()
2246 		       con->peer_name.type, entity_type);  in process_hello()
2247 		con->error_msg = "wrong peer at address";  in process_hello()
2262 		     __func__, con, ceph_pr_addr(my_addr),  in process_hello()
2263 		     ceph_pr_addr(&con->peer_addr));  in process_hello()
2266 		     __func__, con, ceph_pr_addr(my_addr));  in process_hello()
2274 	ret = prepare_auth_request(con);  in process_hello()
2281 	con->state = CEPH_CON_S_V2_AUTH;  in process_hello()
2289 static int process_auth_bad_method(struct ceph_connection *con,  in process_auth_bad_method()  argument
2298 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_bad_method()
2299 		con->error_msg = "protocol error, unexpected auth_bad_method";  in process_auth_bad_method()
2305 	dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,  in process_auth_bad_method()
2315 		dout("%s con %p allowed_protos[%d] %d\n", __func__, con,  in process_auth_bad_method()
2326 		dout("%s con %p allowed_modes[%d] %d\n", __func__, con,  in process_auth_bad_method()
2330 	mutex_unlock(&con->mutex);  in process_auth_bad_method()
2331 	ret = con->ops->handle_auth_bad_method(con, used_proto, result,  in process_auth_bad_method()
2336 	mutex_lock(&con->mutex);  in process_auth_bad_method()
2337 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_bad_method()
2338 		dout("%s con %p state changed to %d\n", __func__, con,  in process_auth_bad_method()
2339 		     con->state);  in process_auth_bad_method()
2343 	dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);  in process_auth_bad_method()
2351 static int process_auth_reply_more(struct ceph_connection *con,  in process_auth_reply_more()  argument
2357 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_reply_more()
2358 		con->error_msg = "protocol error, unexpected auth_reply_more";  in process_auth_reply_more()
2365 	dout("%s con %p payload_len %d\n", __func__, con, payload_len);  in process_auth_reply_more()
2367 	reset_out_kvecs(con);  in process_auth_reply_more()
2368 	ret = prepare_auth_request_more(con, p, payload_len);  in process_auth_reply_more()
2388 static int process_auth_done(struct ceph_connection *con, void *p, void *end)  in process_auth_done()  argument
2399 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_done()
2400 		con->error_msg = "protocol error, unexpected auth_done";  in process_auth_done()
2405 	ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);  in process_auth_done()
2409 	     __func__, con, global_id, con->v2.con_mode, payload_len);  in process_auth_done()
2411 	mutex_unlock(&con->mutex);  in process_auth_done()
2414 	ret = con->ops->handle_auth_done(con, global_id, p, payload_len,  in process_auth_done()
2417 	mutex_lock(&con->mutex);  in process_auth_done()
2418 	if (con->state != CEPH_CON_S_V2_AUTH) {  in process_auth_done()
2419 		dout("%s con %p state changed to %d\n", __func__, con,  in process_auth_done()
2420 		     con->state);  in process_auth_done()
2425 	dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);  in process_auth_done()
2429 	ret = setup_crypto(con, session_key, session_key_len, con_secret,  in process_auth_done()
2434 	reset_out_kvecs(con);  in process_auth_done()
2435 	ret = prepare_auth_signature(con);  in process_auth_done()
2441 	con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;  in process_auth_done()
2453 static int process_auth_signature(struct ceph_connection *con,  in process_auth_signature()  argument
2459 	if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {  in process_auth_signature()
2460 		con->error_msg = "protocol error, unexpected auth_signature";  in process_auth_signature()
2464 	ret = hmac_sha256(con, con->v2.out_sign_kvecs,  in process_auth_signature()
2465 			  con->v2.out_sign_kvec_cnt, hmac);  in process_auth_signature()
2471 		con->error_msg = "integrity error, bad auth signature";  in process_auth_signature()
2475 	dout("%s con %p auth signature ok\n", __func__, con);  in process_auth_signature()
2478 	if (!con->v2.server_cookie) {  in process_auth_signature()
2479 		ret = prepare_client_ident(con);  in process_auth_signature()
2485 		con->state = CEPH_CON_S_V2_SESSION_CONNECT;  in process_auth_signature()
2487 		ret = prepare_session_reconnect(con);  in process_auth_signature()
2493 		con->state = CEPH_CON_S_V2_SESSION_RECONNECT;  in process_auth_signature()
2503 static int process_server_ident(struct ceph_connection *con,  in process_server_ident()  argument
2506 	struct ceph_client *client = from_msgr(con->msgr);  in process_server_ident()
2515 	if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {  in process_server_ident()
2516 		con->error_msg = "protocol error, unexpected server_ident";  in process_server_ident()
2534 	     __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),  in process_server_ident()
2538 	if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {  in process_server_ident()
2540 		       ceph_pr_addr(&con->peer_addr),  in process_server_ident()
2541 		       le32_to_cpu(con->peer_addr.nonce),  in process_server_ident()
2543 		con->error_msg = "wrong peer at address";  in process_server_ident()
2550 		con->error_msg = "missing required protocol features";  in process_server_ident()
2559 	WARN_ON(!con->peer_name.type);  in process_server_ident()
2560 	con->peer_name.num = cpu_to_le64(global_id);  in process_server_ident()
2561 	con->v2.peer_global_seq = global_seq;  in process_server_ident()
2562 	con->peer_features = features;  in process_server_ident()
2564 	con->v2.server_cookie = cookie;  in process_server_ident()
2567 		ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);  in process_server_ident()
2568 		WARN_ON(con->v2.server_cookie);  in process_server_ident()
2570 		WARN_ON(!con->v2.server_cookie);  in process_server_ident()
2573 	clear_in_sign_kvecs(con);  in process_server_ident()
2574 	clear_out_sign_kvecs(con);  in process_server_ident()
2575 	free_conn_bufs(con);  in process_server_ident()
2576 	con->delay = 0;  /* reset backoff memory */  in process_server_ident()
2578 	con->state = CEPH_CON_S_OPEN;  in process_server_ident()
2579 	con->v2.out_state = OUT_S_GET_NEXT;  in process_server_ident()
2587 static int process_ident_missing_features(struct ceph_connection *con,  in process_ident_missing_features()  argument
2590 	struct ceph_client *client = from_msgr(con->msgr);  in process_ident_missing_features()
2593 	if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {  in process_ident_missing_features()
2594 		con->error_msg = "protocol error, unexpected ident_missing_features";  in process_ident_missing_features()
2601 	con->error_msg = "missing required protocol features";  in process_ident_missing_features()
2609 static int process_session_reconnect_ok(struct ceph_connection *con,  in process_session_reconnect_ok()  argument
2614 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_reconnect_ok()
2615 		con->error_msg = "protocol error, unexpected session_reconnect_ok";  in process_session_reconnect_ok()
2621 	dout("%s con %p seq %llu\n", __func__, con, seq);  in process_session_reconnect_ok()
2622 	ceph_con_discard_requeued(con, seq);  in process_session_reconnect_ok()
2624 	clear_in_sign_kvecs(con);  in process_session_reconnect_ok()
2625 	clear_out_sign_kvecs(con);  in process_session_reconnect_ok()
2626 	free_conn_bufs(con);  in process_session_reconnect_ok()
2627 	con->delay = 0;  /* reset backoff memory */  in process_session_reconnect_ok()
2629 	con->state = CEPH_CON_S_OPEN;  in process_session_reconnect_ok()
2630 	con->v2.out_state = OUT_S_GET_NEXT;  in process_session_reconnect_ok()
2638 static int process_session_retry(struct ceph_connection *con,  in process_session_retry()  argument
2644 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_retry()
2645 		con->error_msg = "protocol error, unexpected session_retry";  in process_session_retry()
2651 	dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);  in process_session_retry()
2652 	WARN_ON(connect_seq <= con->v2.connect_seq);  in process_session_retry()
2653 	con->v2.connect_seq = connect_seq + 1;  in process_session_retry()
2655 	free_conn_bufs(con);  in process_session_retry()
2657 	reset_out_kvecs(con);  in process_session_retry()
2658 	ret = prepare_session_reconnect(con);  in process_session_retry()
2671 static int process_session_retry_global(struct ceph_connection *con,  in process_session_retry_global()  argument
2677 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_retry_global()
2678 		con->error_msg = "protocol error, unexpected session_retry_global";  in process_session_retry_global()
2684 	dout("%s con %p global_seq %llu\n", __func__, con, global_seq);  in process_session_retry_global()
2685 	WARN_ON(global_seq <= con->v2.global_seq);  in process_session_retry_global()
2686 	con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);  in process_session_retry_global()
2688 	free_conn_bufs(con);  in process_session_retry_global()
2690 	reset_out_kvecs(con);  in process_session_retry_global()
2691 	ret = prepare_session_reconnect(con);  in process_session_retry_global()
2704 static int process_session_reset(struct ceph_connection *con,  in process_session_reset()  argument
2710 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_reset()
2711 		con->error_msg = "protocol error, unexpected session_reset";  in process_session_reset()
2717 		con->error_msg = "protocol error, bad session_reset";  in process_session_reset()
2721 	pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),  in process_session_reset()
2722 		ceph_pr_addr(&con->peer_addr));  in process_session_reset()
2723 	ceph_con_reset_session(con);  in process_session_reset()
2725 	mutex_unlock(&con->mutex);  in process_session_reset()
2726 	if (con->ops->peer_reset)  in process_session_reset()
2727 		con->ops->peer_reset(con);  in process_session_reset()
2728 	mutex_lock(&con->mutex);  in process_session_reset()
2729 	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {  in process_session_reset()
2730 		dout("%s con %p state changed to %d\n", __func__, con,  in process_session_reset()
2731 		     con->state);  in process_session_reset()
2735 	free_conn_bufs(con);  in process_session_reset()
2737 	reset_out_kvecs(con);  in process_session_reset()
2738 	ret = prepare_client_ident(con);  in process_session_reset()
2744 	con->state = CEPH_CON_S_V2_SESSION_CONNECT;  in process_session_reset()
2752 static int process_keepalive2_ack(struct ceph_connection *con,  in process_keepalive2_ack()  argument
2755 	if (con->state != CEPH_CON_S_OPEN) {  in process_keepalive2_ack()
2756 		con->error_msg = "protocol error, unexpected keepalive2_ack";  in process_keepalive2_ack()
2761 	ceph_decode_timespec64(&con->last_keepalive_ack, p);  in process_keepalive2_ack()
2763 	dout("%s con %p timestamp %lld.%09ld\n", __func__, con,  in process_keepalive2_ack()
2764 	     con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);  in process_keepalive2_ack()
2773 static int process_ack(struct ceph_connection *con, void *p, void *end)  in process_ack()  argument
2777 	if (con->state != CEPH_CON_S_OPEN) {  in process_ack()
2778 		con->error_msg = "protocol error, unexpected ack";  in process_ack()
2784 	dout("%s con %p seq %llu\n", __func__, con, seq);  in process_ack()
2785 	ceph_con_discard_sent(con, seq);  in process_ack()
2793 static int process_control(struct ceph_connection *con, void *p, void *end)  in process_control()  argument
2795 	int tag = con->v2.in_desc.fd_tag;  in process_control()
2798 	dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));  in process_control()
2802 		ret = process_hello(con, p, end);  in process_control()
2805 		ret = process_auth_bad_method(con, p, end);  in process_control()
2808 		ret = process_auth_reply_more(con, p, end);  in process_control()
2811 		ret = process_auth_done(con, p, end);  in process_control()
2814 		ret = process_auth_signature(con, p, end);  in process_control()
2817 		ret = process_server_ident(con, p, end);  in process_control()
2820 		ret = process_ident_missing_features(con, p, end);  in process_control()
2823 		ret = process_session_reconnect_ok(con, p, end);  in process_control()
2826 		ret = process_session_retry(con, p, end);  in process_control()
2829 		ret = process_session_retry_global(con, p, end);  in process_control()
2832 		ret = process_session_reset(con, p, end);  in process_control()
2835 		ret = process_keepalive2_ack(con, p, end);  in process_control()
2838 		ret = process_ack(con, p, end);  in process_control()
2842 		con->error_msg = "protocol error, bad tag";  in process_control()
2846 		dout("%s con %p error %d\n", __func__, con, ret);  in process_control()
2850 	prepare_read_preamble(con);  in process_control()
2860 static int process_message_header(struct ceph_connection *con,  in process_message_header()  argument
2863 	struct ceph_frame_desc *desc = &con->v2.in_desc;  in process_message_header()
2872 	if ((s64)seq - (s64)con->in_seq < 1) {  in process_message_header()
2874 			ENTITY_NAME(con->peer_name),  in process_message_header()
2875 			ceph_pr_addr(&con->peer_addr),  in process_message_header()
2876 			seq, con->in_seq + 1);  in process_message_header()
2879 	if ((s64)seq - (s64)con->in_seq > 1) {  in process_message_header()
2880 		pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);  in process_message_header()
2881 		con->error_msg = "bad message sequence # for incoming message";  in process_message_header()
2885 	ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));  in process_message_header()
2888 		    desc->fd_lens[3], &con->peer_name);  in process_message_header()
2889 	ret = ceph_con_in_msg_alloc(con, &hdr, &skip);  in process_message_header()
2893 	WARN_ON(!con->in_msg ^ skip);  in process_message_header()
2897 	WARN_ON(!con->in_msg);  in process_message_header()
2898 	WARN_ON(con->in_msg->con != con);  in process_message_header()
2902 static int process_message(struct ceph_connection *con)  in process_message()  argument
2904 	ceph_con_process_message(con);  in process_message()
2910 	if (con->state != CEPH_CON_S_OPEN) {  in process_message()
2911 		dout("%s con %p state changed to %d\n", __func__, con,  in process_message()
2912 		     con->state);  in process_message()
2916 	prepare_read_preamble(con);  in process_message()
2920 static int __handle_control(struct ceph_connection *con, void *p)  in __handle_control()  argument
2922 	void *end = p + con->v2.in_desc.fd_lens[0];  in __handle_control()
2926 	if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)  in __handle_control()
2927 		return process_control(con, p, end);  in __handle_control()
2929 	ret = process_message_header(con, p, end);  in __handle_control()
2933 		prepare_skip_message(con);  in __handle_control()
2937 	msg = con->in_msg;  /* set in process_message_header() */  in __handle_control()
2952 		return process_message(con);  in __handle_control()
2954 	if (con_secure(con))  in __handle_control()
2955 		return prepare_read_tail_secure(con);  in __handle_control()
2957 	return prepare_read_tail_plain(con);  in __handle_control()
2960 static int handle_preamble(struct ceph_connection *con)  in handle_preamble()  argument
2962 	struct ceph_frame_desc *desc = &con->v2.in_desc;  in handle_preamble()
2965 	if (con_secure(con)) {  in handle_preamble()
2966 		ret = decrypt_preamble(con);  in handle_preamble()
2969 				con->error_msg = "integrity error, bad preamble auth tag";  in handle_preamble()
2974 	ret = decode_preamble(con->v2.in_buf, desc);  in handle_preamble()
2977 			con->error_msg = "integrity error, bad crc";  in handle_preamble()
2979 			con->error_msg = "protocol error, bad preamble";  in handle_preamble()
2984 	     con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],  in handle_preamble()
2987 	if (!con_secure(con))  in handle_preamble()
2988 		return prepare_read_control(con);  in handle_preamble()
2991 		return prepare_read_control_remainder(con);  in handle_preamble()
2993 	return __handle_control(con, CTRL_BODY(con->v2.in_buf));  in handle_preamble()
2996 static int handle_control(struct ceph_connection *con)  in handle_control()  argument
2998 	int ctrl_len = con->v2.in_desc.fd_lens[0];  in handle_control()
3002 	WARN_ON(con_secure(con));  in handle_control()
3004 	ret = verify_control_crc(con);  in handle_control()
3006 		con->error_msg = "integrity error, bad crc";  in handle_control()
3010 	if (con->state == CEPH_CON_S_V2_AUTH) {  in handle_control()
3011 		buf = alloc_conn_buf(con, ctrl_len);  in handle_control()
3015 		memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);  in handle_control()
3016 		return __handle_control(con, buf);  in handle_control()
3019 	return __handle_control(con, con->v2.in_kvecs[0].iov_base);  in handle_control()
3022 static int handle_control_remainder(struct ceph_connection *con)  in handle_control_remainder()  argument
3026 	WARN_ON(!con_secure(con));  in handle_control_remainder()
3028 	ret = decrypt_control_remainder(con);  in handle_control_remainder()
3031 			con->error_msg = "integrity error, bad control remainder auth tag";  in handle_control_remainder()
3035 	return __handle_control(con, con->v2.in_kvecs[0].iov_base -  in handle_control_remainder()
3039 static int handle_epilogue(struct ceph_connection *con)  in handle_epilogue()  argument
3044 	if (con_secure(con)) {  in handle_epilogue()
3045 		ret = decrypt_tail(con);  in handle_epilogue()
3048 				con->error_msg = "integrity error, bad epilogue auth tag";  in handle_epilogue()
3053 		ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);  in handle_epilogue()
3055 			con->error_msg = "protocol error, bad epilogue";  in handle_epilogue()
3059 		ret = decode_epilogue(con->v2.in_buf, &front_crc,  in handle_epilogue()
3062 			con->error_msg = "protocol error, bad epilogue";  in handle_epilogue()
3066 		ret = verify_epilogue_crcs(con, front_crc, middle_crc,  in handle_epilogue()
3069 			con->error_msg = "integrity error, bad crc";  in handle_epilogue()
3074 	return process_message(con);  in handle_epilogue()
3077 static void finish_skip(struct ceph_connection *con)  in finish_skip()  argument
3079 	dout("%s con %p\n", __func__, con);  in finish_skip()
3081 	if (con_secure(con))  in finish_skip()
3082 		gcm_inc_nonce(&con->v2.in_gcm_nonce);  in finish_skip()
3084 	__finish_skip(con);  in finish_skip()
3087 static int populate_in_iter(struct ceph_connection *con)  in populate_in_iter()  argument
3091 	dout("%s con %p state %d in_state %d\n", __func__, con, con->state,  in populate_in_iter()
3092 	     con->v2.in_state);  in populate_in_iter()
3093 	WARN_ON(iov_iter_count(&con->v2.in_iter));  in populate_in_iter()
3095 	if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {  in populate_in_iter()
3096 		ret = process_banner_prefix(con);  in populate_in_iter()
3097 	} else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {  in populate_in_iter()
3098 		ret = process_banner_payload(con);  in populate_in_iter()
3099 	} else if ((con->state >= CEPH_CON_S_V2_HELLO &&  in populate_in_iter()
3100 		    con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||  in populate_in_iter()
3101 		   con->state == CEPH_CON_S_OPEN) {  in populate_in_iter()
3102 		switch (con->v2.in_state) {  in populate_in_iter()
3104 			ret = handle_preamble(con);  in populate_in_iter()
3107 			ret = handle_control(con);  in populate_in_iter()
3110 			ret = handle_control_remainder(con);  in populate_in_iter()
3113 			ret = prepare_read_data(con);  in populate_in_iter()
3116 			prepare_read_data_cont(con);  in populate_in_iter()
3120 			prepare_read_enc_page(con);  in populate_in_iter()
3124 			ret = prepare_sparse_read_data(con);  in populate_in_iter()
3127 			ret = prepare_sparse_read_cont(con);  in populate_in_iter()
3130 			ret = handle_epilogue(con);  in populate_in_iter()
3133 			finish_skip(con);  in populate_in_iter()
3137 			WARN(1, "bad in_state %d", con->v2.in_state);  in populate_in_iter()
3141 		WARN(1, "bad state %d", con->state);  in populate_in_iter()
3145 		dout("%s con %p error %d\n", __func__, con, ret);  in populate_in_iter()
3149 	if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))  in populate_in_iter()
3151 	dout("%s con %p populated %zu\n", __func__, con,  in populate_in_iter()
3152 	     iov_iter_count(&con->v2.in_iter));  in populate_in_iter()
3156 int ceph_con_v2_try_read(struct ceph_connection *con)  in ceph_con_v2_try_read()  argument
3160 	dout("%s con %p state %d need %zu\n", __func__, con, con->state,  in ceph_con_v2_try_read()
3161 	     iov_iter_count(&con->v2.in_iter));  in ceph_con_v2_try_read()
3163 	if (con->state == CEPH_CON_S_PREOPEN)  in ceph_con_v2_try_read()
3171 	if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))  in ceph_con_v2_try_read()
3175 		ret = ceph_tcp_recv(con);  in ceph_con_v2_try_read()
3179 		ret = populate_in_iter(con);  in ceph_con_v2_try_read()
3181 			if (ret && ret != -EAGAIN && !con->error_msg)  in ceph_con_v2_try_read()
3182 				con->error_msg = "read processing error";  in ceph_con_v2_try_read()
3188 static void queue_data(struct ceph_connection *con)  in queue_data()  argument
3192 	con->v2.out_epil.data_crc = -1;  in queue_data()
3193 	ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,  in queue_data()
3194 				  data_len(con->out_msg));  in queue_data()
3196 	get_bvec_at(&con->v2.out_cursor, &bv);  in queue_data()
3197 	set_out_bvec(con, &bv, true);  in queue_data()
3198 	con->v2.out_state = OUT_S_QUEUE_DATA_CONT;  in queue_data()
3201 static void queue_data_cont(struct ceph_connection *con)  in queue_data_cont()  argument
3205 	con->v2.out_epil.data_crc = ceph_crc32c_page(  in queue_data_cont()
3206 		con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,  in queue_data_cont()
3207 		con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);  in queue_data_cont()
3209 	ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);  in queue_data_cont()
3210 	if (con->v2.out_cursor.total_resid) {  in queue_data_cont()
3211 		get_bvec_at(&con->v2.out_cursor, &bv);  in queue_data_cont()
3212 		set_out_bvec(con, &bv, true);  in queue_data_cont()
3213 		WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);  in queue_data_cont()
3221 	reset_out_kvecs(con);  in queue_data_cont()
3222 	prepare_epilogue_plain(con, false);  in queue_data_cont()
3223 	con->v2.out_state = OUT_S_FINISH_MESSAGE;  in queue_data_cont()
3226 static void queue_enc_page(struct ceph_connection *con)  in queue_enc_page()  argument
3230 	dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,  in queue_enc_page()
3231 	     con->v2.out_enc_resid);  in queue_enc_page()
3232 	WARN_ON(!con->v2.out_enc_resid);  in queue_enc_page()
3234 	bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i],  in queue_enc_page()
3235 		      min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0);  in queue_enc_page()
3237 	set_out_bvec(con, &bv, false);  in queue_enc_page()
3238 	con->v2.out_enc_i++;  in queue_enc_page()
3239 	con->v2.out_enc_resid -= bv.bv_len;  in queue_enc_page()
3241 	if (con->v2.out_enc_resid) {  in queue_enc_page()
3242 		WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);  in queue_enc_page()
3250 	WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);  in queue_enc_page()
3251 	con->v2.out_state = OUT_S_FINISH_MESSAGE;  in queue_enc_page()
3254 static void queue_zeros(struct ceph_connection *con)  in queue_zeros()  argument
3256 	dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);  in queue_zeros()
3258 	if (con->v2.out_zero) {  in queue_zeros()
3259 		set_out_bvec_zero(con);  in queue_zeros()
3260 		con->v2.out_zero -= con->v2.out_bvec.bv_len;  in queue_zeros()
3261 		con->v2.out_state = OUT_S_QUEUE_ZEROS;  in queue_zeros()
3270 	reset_out_kvecs(con);  in queue_zeros()
3271 	prepare_epilogue_plain(con, true);  in queue_zeros()
3272 	con->v2.out_state = OUT_S_FINISH_MESSAGE;  in queue_zeros()
3275 static void finish_message(struct ceph_connection *con)  in finish_message()  argument
3277 	dout("%s con %p msg %p\n", __func__, con, con->out_msg);  in finish_message()
3280 	if (con->v2.out_enc_pages) {  in finish_message()
3281 		WARN_ON(!con->v2.out_enc_page_cnt);  in finish_message()
3282 		ceph_release_page_vector(con->v2.out_enc_pages,  in finish_message()
3283 					 con->v2.out_enc_page_cnt);  in finish_message()
3284 		con->v2.out_enc_pages = NULL;  in finish_message()
3285 		con->v2.out_enc_page_cnt = 0;  in finish_message()
3288 	if (con->out_msg) {  in finish_message()
3289 		ceph_msg_put(con->out_msg);  in finish_message()
3290 		con->out_msg = NULL;  in finish_message()
3293 	con->v2.out_state = OUT_S_GET_NEXT;  in finish_message()
3296 static int populate_out_iter(struct ceph_connection *con)  in populate_out_iter()  argument
3300 	dout("%s con %p state %d out_state %d\n", __func__, con, con->state,  in populate_out_iter()
3301 	     con->v2.out_state);  in populate_out_iter()
3302 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in populate_out_iter()
3304 	if (con->state != CEPH_CON_S_OPEN) {  in populate_out_iter()
3305 		WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||  in populate_out_iter()
3306 			con->state > CEPH_CON_S_V2_SESSION_RECONNECT);  in populate_out_iter()
3310 	switch (con->v2.out_state) {  in populate_out_iter()
3312 		WARN_ON(!con->out_msg);  in populate_out_iter()
3313 		queue_data(con);  in populate_out_iter()
3316 		WARN_ON(!con->out_msg);  in populate_out_iter()
3317 		queue_data_cont(con);  in populate_out_iter()
3320 		queue_enc_page(con);  in populate_out_iter()
3323 		WARN_ON(con->out_msg);  /* revoked */  in populate_out_iter()
3324 		queue_zeros(con);  in populate_out_iter()
3327 		finish_message(con);  in populate_out_iter()
3332 		WARN(1, "bad out_state %d", con->v2.out_state);  in populate_out_iter()
3336 	WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);  in populate_out_iter()
3337 	if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {  in populate_out_iter()
3338 		ret = prepare_keepalive2(con);  in populate_out_iter()
3343 	} else if (!list_empty(&con->out_queue)) {  in populate_out_iter()
3344 		ceph_con_get_out_msg(con);  in populate_out_iter()
3345 		ret = prepare_message(con);  in populate_out_iter()
3350 	} else if (con->in_seq > con->in_seq_acked) {  in populate_out_iter()
3351 		ret = prepare_ack(con);  in populate_out_iter()
3361 	if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))  in populate_out_iter()
3363 	dout("%s con %p populated %zu\n", __func__, con,  in populate_out_iter()
3364 	     iov_iter_count(&con->v2.out_iter));  in populate_out_iter()
3368 	WARN_ON(iov_iter_count(&con->v2.out_iter));  in populate_out_iter()
3369 	dout("%s con %p nothing pending\n", __func__, con);  in populate_out_iter()
3370 	ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);  in populate_out_iter()
3374 int ceph_con_v2_try_write(struct ceph_connection *con)  in ceph_con_v2_try_write()  argument
3378 	dout("%s con %p state %d have %zu\n", __func__, con, con->state,  in ceph_con_v2_try_write()
3379 	     iov_iter_count(&con->v2.out_iter));  in ceph_con_v2_try_write()
3382 	if (con->state == CEPH_CON_S_PREOPEN) {  in ceph_con_v2_try_write()
3383 		WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);  in ceph_con_v2_try_write()
3390 		con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);  in ceph_con_v2_try_write()
3391 		if (con->v2.server_cookie)  in ceph_con_v2_try_write()
3392 			con->v2.connect_seq++;  in ceph_con_v2_try_write()
3394 		ret = prepare_read_banner_prefix(con);  in ceph_con_v2_try_write()
3397 			con->error_msg = "connect error";  in ceph_con_v2_try_write()
3401 		reset_out_kvecs(con);  in ceph_con_v2_try_write()
3402 		ret = prepare_banner(con);  in ceph_con_v2_try_write()
3405 			con->error_msg = "connect error";  in ceph_con_v2_try_write()
3409 		ret = ceph_tcp_connect(con);  in ceph_con_v2_try_write()
3412 			con->error_msg = "connect error";  in ceph_con_v2_try_write()
3417 	if (!iov_iter_count(&con->v2.out_iter)) {  in ceph_con_v2_try_write()
3418 		ret = populate_out_iter(con);  in ceph_con_v2_try_write()
3420 			if (ret && ret != -EAGAIN && !con->error_msg)  in ceph_con_v2_try_write()
3421 				con->error_msg = "write processing error";  in ceph_con_v2_try_write()
3426 	tcp_sock_set_cork(con->sock->sk, true);  in ceph_con_v2_try_write()
3428 		ret = ceph_tcp_send(con);  in ceph_con_v2_try_write()
3432 		ret = populate_out_iter(con);  in ceph_con_v2_try_write()
3434 			if (ret && ret != -EAGAIN && !con->error_msg)  in ceph_con_v2_try_write()
3435 				con->error_msg = "write processing error";  in ceph_con_v2_try_write()
3440 	tcp_sock_set_cork(con->sock->sk, false);  in ceph_con_v2_try_write()
3457 static void prepare_zero_front(struct ceph_connection *con, int resid)  in prepare_zero_front()  argument
3461 	WARN_ON(!resid || resid > front_len(con->out_msg));  in prepare_zero_front()
3462 	sent = front_len(con->out_msg) - resid;  in prepare_zero_front()
3463 	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);  in prepare_zero_front()
3466 		con->v2.out_epil.front_crc =  in prepare_zero_front()
3467 			crc32c(-1, con->out_msg->front.iov_base, sent);  in prepare_zero_front()
3468 		con->v2.out_epil.front_crc =  in prepare_zero_front()
3469 			crc32c_zeros(con->v2.out_epil.front_crc, resid);  in prepare_zero_front()
3471 		con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);  in prepare_zero_front()
3474 	con->v2.out_iter.count -= resid;  in prepare_zero_front()
3475 	out_zero_add(con, resid);  in prepare_zero_front()
3478 static void prepare_zero_middle(struct ceph_connection *con, int resid)  in prepare_zero_middle()  argument
3482 	WARN_ON(!resid || resid > middle_len(con->out_msg));  in prepare_zero_middle()
3483 	sent = middle_len(con->out_msg) - resid;  in prepare_zero_middle()
3484 	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);  in prepare_zero_middle()
3487 		con->v2.out_epil.middle_crc =  in prepare_zero_middle()
3488 			crc32c(-1, con->out_msg->middle->vec.iov_base, sent);  in prepare_zero_middle()
3489 		con->v2.out_epil.middle_crc =  in prepare_zero_middle()
3490 			crc32c_zeros(con->v2.out_epil.middle_crc, resid);  in prepare_zero_middle()
3492 		con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);  in prepare_zero_middle()
3495 	con->v2.out_iter.count -= resid;  in prepare_zero_middle()
3496 	out_zero_add(con, resid);  in prepare_zero_middle()
3499 static void prepare_zero_data(struct ceph_connection *con)  in prepare_zero_data()  argument
3501 	dout("%s con %p\n", __func__, con);  in prepare_zero_data()
3502 	con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));  in prepare_zero_data()
3503 	out_zero_add(con, data_len(con->out_msg));  in prepare_zero_data()
3506 static void revoke_at_queue_data(struct ceph_connection *con)  in revoke_at_queue_data()  argument
3511 	WARN_ON(!data_len(con->out_msg));  in revoke_at_queue_data()
3512 	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));  in revoke_at_queue_data()
3513 	resid = iov_iter_count(&con->v2.out_iter);  in revoke_at_queue_data()
3515 	boundary = front_len(con->out_msg) + middle_len(con->out_msg);  in revoke_at_queue_data()
3519 		dout("%s con %p was sending head\n", __func__, con);  in revoke_at_queue_data()
3520 		if (front_len(con->out_msg))  in revoke_at_queue_data()
3521 			prepare_zero_front(con, front_len(con->out_msg));  in revoke_at_queue_data()
3522 		if (middle_len(con->out_msg))  in revoke_at_queue_data()
3523 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_queue_data()
3524 		prepare_zero_data(con);  in revoke_at_queue_data()
3525 		WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);  in revoke_at_queue_data()
3526 		con->v2.out_state = OUT_S_QUEUE_ZEROS;  in revoke_at_queue_data()
3530 	boundary = middle_len(con->out_msg);  in revoke_at_queue_data()
3533 		dout("%s con %p was sending front\n", __func__, con);  in revoke_at_queue_data()
3534 		prepare_zero_front(con, resid);  in revoke_at_queue_data()
3535 		if (middle_len(con->out_msg))  in revoke_at_queue_data()
3536 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_queue_data()
3537 		prepare_zero_data(con);  in revoke_at_queue_data()
3538 		queue_zeros(con);  in revoke_at_queue_data()
3543 	dout("%s con %p was sending middle\n", __func__, con);  in revoke_at_queue_data()
3544 	prepare_zero_middle(con, resid);  in revoke_at_queue_data()
3545 	prepare_zero_data(con);  in revoke_at_queue_data()
3546 	queue_zeros(con);  in revoke_at_queue_data()
3549 static void revoke_at_queue_data_cont(struct ceph_connection *con)  in revoke_at_queue_data_cont()  argument
3553 	WARN_ON(!data_len(con->out_msg));  in revoke_at_queue_data_cont()
3554 	WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));  in revoke_at_queue_data_cont()
3555 	resid = iov_iter_count(&con->v2.out_iter);  in revoke_at_queue_data_cont()
3556 	WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);  in revoke_at_queue_data_cont()
3557 	sent = con->v2.out_bvec.bv_len - resid;  in revoke_at_queue_data_cont()
3558 	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);  in revoke_at_queue_data_cont()
3561 		con->v2.out_epil.data_crc = ceph_crc32c_page(  in revoke_at_queue_data_cont()
3562 			con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,  in revoke_at_queue_data_cont()
3563 			con->v2.out_bvec.bv_offset, sent);  in revoke_at_queue_data_cont()
3564 		ceph_msg_data_advance(&con->v2.out_cursor, sent);  in revoke_at_queue_data_cont()
3566 	WARN_ON(resid > con->v2.out_cursor.total_resid);  in revoke_at_queue_data_cont()
3567 	con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,  in revoke_at_queue_data_cont()
3568 						con->v2.out_cursor.total_resid);  in revoke_at_queue_data_cont()
3570 	con->v2.out_iter.count -= resid;  in revoke_at_queue_data_cont()
3571 	out_zero_add(con, con->v2.out_cursor.total_resid);  in revoke_at_queue_data_cont()
3572 	queue_zeros(con);  in revoke_at_queue_data_cont()
3575 static void revoke_at_finish_message(struct ceph_connection *con)  in revoke_at_finish_message()  argument
3580 	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));  in revoke_at_finish_message()
3581 	resid = iov_iter_count(&con->v2.out_iter);  in revoke_at_finish_message()
3583 	if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&  in revoke_at_finish_message()
3584 	    !data_len(con->out_msg)) {  in revoke_at_finish_message()
3587 		     __func__, con);  in revoke_at_finish_message()
3591 	boundary = front_len(con->out_msg) + middle_len(con->out_msg) +  in revoke_at_finish_message()
3596 		dout("%s con %p was sending head\n", __func__, con);  in revoke_at_finish_message()
3597 		if (front_len(con->out_msg))  in revoke_at_finish_message()
3598 			prepare_zero_front(con, front_len(con->out_msg));  in revoke_at_finish_message()
3599 		if (middle_len(con->out_msg))  in revoke_at_finish_message()
3600 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_finish_message()
3601 		con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3602 		WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);  in revoke_at_finish_message()
3603 		con->v2.out_state = OUT_S_QUEUE_ZEROS;  in revoke_at_finish_message()
3607 	boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3610 		dout("%s con %p was sending front\n", __func__, con);  in revoke_at_finish_message()
3611 		prepare_zero_front(con, resid);  in revoke_at_finish_message()
3612 		if (middle_len(con->out_msg))  in revoke_at_finish_message()
3613 			prepare_zero_middle(con, middle_len(con->out_msg));  in revoke_at_finish_message()
3614 		con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3615 		queue_zeros(con);  in revoke_at_finish_message()
3622 		dout("%s con %p was sending middle\n", __func__, con);  in revoke_at_finish_message()
3623 		prepare_zero_middle(con, resid);  in revoke_at_finish_message()
3624 		con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_finish_message()
3625 		queue_zeros(con);  in revoke_at_finish_message()
3630 	dout("%s con %p was sending epilogue - noop\n", __func__, con);  in revoke_at_finish_message()
3633 void ceph_con_v2_revoke(struct ceph_connection *con)  in ceph_con_v2_revoke()  argument
3635 	WARN_ON(con->v2.out_zero);  in ceph_con_v2_revoke()
3637 	if (con_secure(con)) {  in ceph_con_v2_revoke()
3638 		WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&  in ceph_con_v2_revoke()
3639 			con->v2.out_state != OUT_S_FINISH_MESSAGE);  in ceph_con_v2_revoke()
3640 		dout("%s con %p secure - noop\n", __func__, con);  in ceph_con_v2_revoke()
3644 	switch (con->v2.out_state) {  in ceph_con_v2_revoke()
3646 		revoke_at_queue_data(con);  in ceph_con_v2_revoke()
3649 		revoke_at_queue_data_cont(con);  in ceph_con_v2_revoke()
3652 		revoke_at_finish_message(con);  in ceph_con_v2_revoke()
3655 		WARN(1, "bad out_state %d", con->v2.out_state);  in ceph_con_v2_revoke()
3660 static void revoke_at_prepare_read_data(struct ceph_connection *con)  in revoke_at_prepare_read_data()  argument
3665 	WARN_ON(con_secure(con));  in revoke_at_prepare_read_data()
3666 	WARN_ON(!data_len(con->in_msg));  in revoke_at_prepare_read_data()
3667 	WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));  in revoke_at_prepare_read_data()
3668 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_read_data()
3671 	remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;  in revoke_at_prepare_read_data()
3672 	dout("%s con %p resid %d remaining %d\n", __func__, con, resid,  in revoke_at_prepare_read_data()
3674 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_read_data()
3675 	set_in_skip(con, resid + remaining);  in revoke_at_prepare_read_data()
3676 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_read_data()
3679 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)  in revoke_at_prepare_read_data_cont()  argument
3684 	WARN_ON(con_secure(con));  in revoke_at_prepare_read_data_cont()
3685 	WARN_ON(!data_len(con->in_msg));  in revoke_at_prepare_read_data_cont()
3686 	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));  in revoke_at_prepare_read_data_cont()
3687 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_read_data_cont()
3688 	WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);  in revoke_at_prepare_read_data_cont()
3689 	recved = con->v2.in_bvec.bv_len - resid;  in revoke_at_prepare_read_data_cont()
3690 	dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);  in revoke_at_prepare_read_data_cont()
3693 		ceph_msg_data_advance(&con->v2.in_cursor, recved);  in revoke_at_prepare_read_data_cont()
3694 	WARN_ON(resid > con->v2.in_cursor.total_resid);  in revoke_at_prepare_read_data_cont()
3697 	dout("%s con %p total_resid %zu remaining %d\n", __func__, con,  in revoke_at_prepare_read_data_cont()
3698 	     con->v2.in_cursor.total_resid, remaining);  in revoke_at_prepare_read_data_cont()
3699 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_read_data_cont()
3700 	set_in_skip(con, con->v2.in_cursor.total_resid + remaining);  in revoke_at_prepare_read_data_cont()
3701 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_read_data_cont()
3704 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)  in revoke_at_prepare_read_enc_page()  argument
3708 	WARN_ON(!con_secure(con));  in revoke_at_prepare_read_enc_page()
3709 	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));  in revoke_at_prepare_read_enc_page()
3710 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_read_enc_page()
3711 	WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);  in revoke_at_prepare_read_enc_page()
3713 	dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,  in revoke_at_prepare_read_enc_page()
3714 	     con->v2.in_enc_resid);  in revoke_at_prepare_read_enc_page()
3715 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_read_enc_page()
3716 	set_in_skip(con, resid + con->v2.in_enc_resid);  in revoke_at_prepare_read_enc_page()
3717 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_read_enc_page()
3720 static void revoke_at_prepare_sparse_data(struct ceph_connection *con)  in revoke_at_prepare_sparse_data()  argument
3725 	WARN_ON(con_secure(con));  in revoke_at_prepare_sparse_data()
3726 	WARN_ON(!data_len(con->in_msg));  in revoke_at_prepare_sparse_data()
3727 	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));  in revoke_at_prepare_sparse_data()
3728 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_prepare_sparse_data()
3729 	dout("%s con %p resid %d\n", __func__, con, resid);  in revoke_at_prepare_sparse_data()
3731 	remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain;  in revoke_at_prepare_sparse_data()
3732 	con->v2.in_iter.count -= resid;  in revoke_at_prepare_sparse_data()
3733 	set_in_skip(con, resid + remaining);  in revoke_at_prepare_sparse_data()
3734 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_prepare_sparse_data()
3737 static void revoke_at_handle_epilogue(struct ceph_connection *con)  in revoke_at_handle_epilogue()  argument
3741 	resid = iov_iter_count(&con->v2.in_iter);  in revoke_at_handle_epilogue()
3744 	dout("%s con %p resid %d\n", __func__, con, resid);  in revoke_at_handle_epilogue()
3745 	con->v2.in_iter.count -= resid;  in revoke_at_handle_epilogue()
3746 	set_in_skip(con, resid);  in revoke_at_handle_epilogue()
3747 	con->v2.in_state = IN_S_FINISH_SKIP;  in revoke_at_handle_epilogue()
3750 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)  in ceph_con_v2_revoke_incoming()  argument
3752 	switch (con->v2.in_state) {  in ceph_con_v2_revoke_incoming()
3755 		revoke_at_prepare_read_data(con);  in ceph_con_v2_revoke_incoming()
3758 		revoke_at_prepare_read_data_cont(con);  in ceph_con_v2_revoke_incoming()
3761 		revoke_at_prepare_read_enc_page(con);  in ceph_con_v2_revoke_incoming()
3764 		revoke_at_prepare_sparse_data(con);  in ceph_con_v2_revoke_incoming()
3767 		revoke_at_handle_epilogue(con);  in ceph_con_v2_revoke_incoming()
3770 		WARN(1, "bad in_state %d", con->v2.in_state);  in ceph_con_v2_revoke_incoming()
3775 bool ceph_con_v2_opened(struct ceph_connection *con)  in ceph_con_v2_opened()  argument
3777 	return con->v2.peer_global_seq;  in ceph_con_v2_opened()
3780 void ceph_con_v2_reset_session(struct ceph_connection *con)  in ceph_con_v2_reset_session()  argument
3782 	con->v2.client_cookie = 0;  in ceph_con_v2_reset_session()
3783 	con->v2.server_cookie = 0;  in ceph_con_v2_reset_session()
3784 	con->v2.global_seq = 0;  in ceph_con_v2_reset_session()
3785 	con->v2.connect_seq = 0;  in ceph_con_v2_reset_session()
3786 	con->v2.peer_global_seq = 0;  in ceph_con_v2_reset_session()
3789 void ceph_con_v2_reset_protocol(struct ceph_connection *con)  in ceph_con_v2_reset_protocol()  argument
3791 	iov_iter_truncate(&con->v2.in_iter, 0);  in ceph_con_v2_reset_protocol()
3792 	iov_iter_truncate(&con->v2.out_iter, 0);  in ceph_con_v2_reset_protocol()
3793 	con->v2.out_zero = 0;  in ceph_con_v2_reset_protocol()
3795 	clear_in_sign_kvecs(con);  in ceph_con_v2_reset_protocol()
3796 	clear_out_sign_kvecs(con);  in ceph_con_v2_reset_protocol()
3797 	free_conn_bufs(con);  in ceph_con_v2_reset_protocol()
3799 	if (con->v2.in_enc_pages) {  in ceph_con_v2_reset_protocol()
3800 		WARN_ON(!con->v2.in_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3801 		ceph_release_page_vector(con->v2.in_enc_pages,  in ceph_con_v2_reset_protocol()
3802 					 con->v2.in_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3803 		con->v2.in_enc_pages = NULL;  in ceph_con_v2_reset_protocol()
3804 		con->v2.in_enc_page_cnt = 0;  in ceph_con_v2_reset_protocol()
3806 	if (con->v2.out_enc_pages) {  in ceph_con_v2_reset_protocol()
3807 		WARN_ON(!con->v2.out_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3808 		ceph_release_page_vector(con->v2.out_enc_pages,  in ceph_con_v2_reset_protocol()
3809 					 con->v2.out_enc_page_cnt);  in ceph_con_v2_reset_protocol()
3810 		con->v2.out_enc_pages = NULL;  in ceph_con_v2_reset_protocol()
3811 		con->v2.out_enc_page_cnt = 0;  in ceph_con_v2_reset_protocol()
3814 	con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;  in ceph_con_v2_reset_protocol()
3815 	memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);  in ceph_con_v2_reset_protocol()
3816 	memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);  in ceph_con_v2_reset_protocol()
3818 	if (con->v2.hmac_tfm) {  in ceph_con_v2_reset_protocol()
3819 		crypto_free_shash(con->v2.hmac_tfm);  in ceph_con_v2_reset_protocol()
3820 		con->v2.hmac_tfm = NULL;  in ceph_con_v2_reset_protocol()
3822 	if (con->v2.gcm_req) {  in ceph_con_v2_reset_protocol()
3823 		aead_request_free(con->v2.gcm_req);  in ceph_con_v2_reset_protocol()
3824 		con->v2.gcm_req = NULL;  in ceph_con_v2_reset_protocol()
3826 	if (con->v2.gcm_tfm) {  in ceph_con_v2_reset_protocol()
3827 		crypto_free_aead(con->v2.gcm_tfm);  in ceph_con_v2_reset_protocol()
3828 		con->v2.gcm_tfm = NULL;  in ceph_con_v2_reset_protocol()