64 				   struct iscsi_segment *segment);
90 iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,  in iscsi_tcp_segment_init_sg()  argument
93 	segment->sg = sg;  in iscsi_tcp_segment_init_sg()
94 	segment->sg_offset = offset;  in iscsi_tcp_segment_init_sg()
95 	segment->size = min(sg->length - offset,  in iscsi_tcp_segment_init_sg()
96 			    segment->total_size - segment->total_copied);  in iscsi_tcp_segment_init_sg()
97 	segment->data = NULL;  in iscsi_tcp_segment_init_sg()
109 static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)  in iscsi_tcp_segment_map()  argument
113 	if (segment->data != NULL || !segment->sg)  in iscsi_tcp_segment_map()
116 	sg = segment->sg;  in iscsi_tcp_segment_map()
117 	BUG_ON(segment->sg_mapped);  in iscsi_tcp_segment_map()
135 		segment->atomic_mapped = true;  in iscsi_tcp_segment_map()
136 		segment->sg_mapped = kmap_atomic(sg_page(sg));  in iscsi_tcp_segment_map()
138 		segment->atomic_mapped = false;  in iscsi_tcp_segment_map()
140 		segment->sg_mapped = kmap(sg_page(sg));  in iscsi_tcp_segment_map()
143 	segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;  in iscsi_tcp_segment_map()
146 void iscsi_tcp_segment_unmap(struct iscsi_segment *segment)  in iscsi_tcp_segment_unmap()  argument
148 	if (segment->sg_mapped) {  in iscsi_tcp_segment_unmap()
149 		if (segment->atomic_mapped)  in iscsi_tcp_segment_unmap()
150 			kunmap_atomic(segment->sg_mapped);  in iscsi_tcp_segment_unmap()
152 			kunmap(sg_page(segment->sg));  in iscsi_tcp_segment_unmap()
153 		segment->sg_mapped = NULL;  in iscsi_tcp_segment_unmap()
154 		segment->data = NULL;  in iscsi_tcp_segment_unmap()
163 iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)  in iscsi_tcp_segment_splice_digest()  argument
165 	segment->data = digest;  in iscsi_tcp_segment_splice_digest()
166 	segment->digest_len = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
167 	segment->total_size += ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
168 	segment->size = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
169 	segment->copied = 0;  in iscsi_tcp_segment_splice_digest()
170 	segment->sg = NULL;  in iscsi_tcp_segment_splice_digest()
171 	segment->hash = NULL;  in iscsi_tcp_segment_splice_digest()
191 			   struct iscsi_segment *segment, int recv,  in iscsi_tcp_segment_done()  argument
198 		      segment->copied, copied, segment->size,  in iscsi_tcp_segment_done()
200 	if (segment->hash && copied) {  in iscsi_tcp_segment_done()
205 		iscsi_tcp_segment_unmap(segment);  in iscsi_tcp_segment_done()
207 		if (!segment->data) {  in iscsi_tcp_segment_done()
209 			sg_set_page(&sg, sg_page(segment->sg), copied,  in iscsi_tcp_segment_done()
210 				    segment->copied + segment->sg_offset +  in iscsi_tcp_segment_done()
211 							segment->sg->offset);  in iscsi_tcp_segment_done()
213 			sg_init_one(&sg, segment->data + segment->copied,  in iscsi_tcp_segment_done()
215 		ahash_request_set_crypt(segment->hash, &sg, NULL, copied);  in iscsi_tcp_segment_done()
216 		crypto_ahash_update(segment->hash);  in iscsi_tcp_segment_done()
219 	segment->copied += copied;  in iscsi_tcp_segment_done()
220 	if (segment->copied < segment->size) {  in iscsi_tcp_segment_done()
221 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
225 	segment->total_copied += segment->copied;  in iscsi_tcp_segment_done()
226 	segment->copied = 0;  in iscsi_tcp_segment_done()
227 	segment->size = 0;  in iscsi_tcp_segment_done()
230 	iscsi_tcp_segment_unmap(segment);  in iscsi_tcp_segment_done()
234 		      segment->total_copied, segment->total_size);  in iscsi_tcp_segment_done()
235 	if (segment->total_copied < segment->total_size) {  in iscsi_tcp_segment_done()
237 		iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),  in iscsi_tcp_segment_done()
239 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
240 		BUG_ON(segment->size == 0);  in iscsi_tcp_segment_done()
246 		pad = iscsi_padding(segment->total_copied);  in iscsi_tcp_segment_done()
250 			segment->total_size += pad;  in iscsi_tcp_segment_done()
251 			segment->size = pad;  in iscsi_tcp_segment_done()
252 			segment->data = segment->padbuf;  in iscsi_tcp_segment_done()
261 	if (segment->hash) {  in iscsi_tcp_segment_done()
262 		ahash_request_set_crypt(segment->hash, NULL,  in iscsi_tcp_segment_done()
263 					segment->digest, 0);  in iscsi_tcp_segment_done()
264 		crypto_ahash_final(segment->hash);  in iscsi_tcp_segment_done()
265 		iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_segment_done()
266 				 recv ? segment->recv_digest : segment->digest);  in iscsi_tcp_segment_done()
293 		       struct iscsi_segment *segment, const void *ptr,  in iscsi_tcp_segment_recv()  argument
298 	while (!iscsi_tcp_segment_done(tcp_conn, segment, 1, copy)) {  in iscsi_tcp_segment_recv()
305 		copy = min(len - copied, segment->size - segment->copied);  in iscsi_tcp_segment_recv()
307 		memcpy(segment->data + segment->copied, ptr + copied, copy);  in iscsi_tcp_segment_recv()
327 		      struct iscsi_segment *segment)  in iscsi_tcp_dgst_verify()  argument
329 	if (!segment->digest_len)  in iscsi_tcp_dgst_verify()
332 	if (memcmp(segment->recv_digest, segment->digest,  in iscsi_tcp_dgst_verify()
333 		   segment->digest_len)) {  in iscsi_tcp_dgst_verify()
345 __iscsi_segment_init(struct iscsi_segment *segment, size_t size,  in __iscsi_segment_init()  argument
348 	memset(segment, 0, sizeof(*segment));  in __iscsi_segment_init()
349 	segment->total_size = size;  in __iscsi_segment_init()
350 	segment->done = done;  in __iscsi_segment_init()
353 		segment->hash = hash;  in __iscsi_segment_init()
359 iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,  in iscsi_segment_init_linear()  argument
363 	__iscsi_segment_init(segment, size, done, hash);  in iscsi_segment_init_linear()
364 	segment->data = data;  in iscsi_segment_init_linear()
365 	segment->size = size;  in iscsi_segment_init_linear()
370 iscsi_segment_seek_sg(struct iscsi_segment *segment,  in iscsi_segment_seek_sg()  argument
379 	__iscsi_segment_init(segment, size, done, hash);  in iscsi_segment_seek_sg()
382 			iscsi_tcp_segment_init_sg(segment, sg, offset);  in iscsi_segment_seek_sg()
405 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_hdr_recv_prep()
416 			 struct iscsi_segment *segment)  in iscsi_tcp_data_recv_done()  argument
421 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_data_recv_done()
443 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_data_recv_prep()
654 			  struct iscsi_segment *segment)  in iscsi_tcp_process_data_in()  argument
660 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_process_data_in()
749 			rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,  in iscsi_tcp_hdr_dissect()
836 			struct iscsi_segment *segment)  in iscsi_tcp_hdr_recv_done()  argument
846 	if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {  in iscsi_tcp_hdr_recv_done()
856 		segment->total_size += ahslen;  in iscsi_tcp_hdr_recv_done()
857 		segment->size += ahslen;  in iscsi_tcp_hdr_recv_done()
866 		if (segment->digest_len == 0) {  in iscsi_tcp_hdr_recv_done()
872 			iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_hdr_recv_done()
873 							segment->recv_digest);  in iscsi_tcp_hdr_recv_done()
878 				      segment->total_copied - ISCSI_DIGEST_SIZE,  in iscsi_tcp_hdr_recv_done()
879 				      segment->digest);  in iscsi_tcp_hdr_recv_done()
881 		if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_hdr_recv_done()
898 	return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;  in iscsi_tcp_recv_segment_is_hdr()
917 	struct iscsi_segment *segment = &tcp_conn->in.segment;  in iscsi_tcp_recv_skb()  local
937 		segment->total_copied = segment->total_size;  in iscsi_tcp_recv_skb()
953 		BUG_ON(segment->copied >= segment->size);  in iscsi_tcp_recv_skb()
957 		rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);  in iscsi_tcp_recv_skb()
961 		if (segment->total_copied >= segment->total_size) {  in iscsi_tcp_recv_skb()
970 	rc = segment->done(tcp_conn, segment);  in iscsi_tcp_recv_skb()