71 				   struct iscsi_segment *segment);
97 iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,  in iscsi_tcp_segment_init_sg()  argument
100 	segment->sg = sg;  in iscsi_tcp_segment_init_sg()
101 	segment->sg_offset = offset;  in iscsi_tcp_segment_init_sg()
102 	segment->size = min(sg->length - offset,  in iscsi_tcp_segment_init_sg()
103 			    segment->total_size - segment->total_copied);  in iscsi_tcp_segment_init_sg()
104 	segment->data = NULL;  in iscsi_tcp_segment_init_sg()
116 static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)  in iscsi_tcp_segment_map()  argument
120 	if (segment->data != NULL || !segment->sg)  in iscsi_tcp_segment_map()
123 	sg = segment->sg;  in iscsi_tcp_segment_map()
124 	BUG_ON(segment->sg_mapped);  in iscsi_tcp_segment_map()
137 		segment->atomic_mapped = true;  in iscsi_tcp_segment_map()
138 		segment->sg_mapped = kmap_atomic(sg_page(sg));  in iscsi_tcp_segment_map()
140 		segment->atomic_mapped = false;  in iscsi_tcp_segment_map()
142 		segment->sg_mapped = kmap(sg_page(sg));  in iscsi_tcp_segment_map()
145 	segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;  in iscsi_tcp_segment_map()
148 void iscsi_tcp_segment_unmap(struct iscsi_segment *segment)  in iscsi_tcp_segment_unmap()  argument
150 	if (segment->sg_mapped) {  in iscsi_tcp_segment_unmap()
151 		if (segment->atomic_mapped)  in iscsi_tcp_segment_unmap()
152 			kunmap_atomic(segment->sg_mapped);  in iscsi_tcp_segment_unmap()
154 			kunmap(sg_page(segment->sg));  in iscsi_tcp_segment_unmap()
155 		segment->sg_mapped = NULL;  in iscsi_tcp_segment_unmap()
156 		segment->data = NULL;  in iscsi_tcp_segment_unmap()
165 iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)  in iscsi_tcp_segment_splice_digest()  argument
167 	segment->data = digest;  in iscsi_tcp_segment_splice_digest()
168 	segment->digest_len = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
169 	segment->total_size += ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
170 	segment->size = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
171 	segment->copied = 0;  in iscsi_tcp_segment_splice_digest()
172 	segment->sg = NULL;  in iscsi_tcp_segment_splice_digest()
173 	segment->hash = NULL;  in iscsi_tcp_segment_splice_digest()
193 			   struct iscsi_segment *segment, int recv,  in iscsi_tcp_segment_done()  argument
200 		      segment->copied, copied, segment->size,  in iscsi_tcp_segment_done()
202 	if (segment->hash && copied) {  in iscsi_tcp_segment_done()
207 		iscsi_tcp_segment_unmap(segment);  in iscsi_tcp_segment_done()
209 		if (!segment->data) {  in iscsi_tcp_segment_done()
211 			sg_set_page(&sg, sg_page(segment->sg), copied,  in iscsi_tcp_segment_done()
212 				    segment->copied + segment->sg_offset +  in iscsi_tcp_segment_done()
213 							segment->sg->offset);  in iscsi_tcp_segment_done()
215 			sg_init_one(&sg, segment->data + segment->copied,  in iscsi_tcp_segment_done()
217 		ahash_request_set_crypt(segment->hash, &sg, NULL, copied);  in iscsi_tcp_segment_done()
218 		crypto_ahash_update(segment->hash);  in iscsi_tcp_segment_done()
221 	segment->copied += copied;  in iscsi_tcp_segment_done()
222 	if (segment->copied < segment->size) {  in iscsi_tcp_segment_done()
223 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
227 	segment->total_copied += segment->copied;  in iscsi_tcp_segment_done()
228 	segment->copied = 0;  in iscsi_tcp_segment_done()
229 	segment->size = 0;  in iscsi_tcp_segment_done()
232 	iscsi_tcp_segment_unmap(segment);  in iscsi_tcp_segment_done()
236 		      segment->total_copied, segment->total_size);  in iscsi_tcp_segment_done()
237 	if (segment->total_copied < segment->total_size) {  in iscsi_tcp_segment_done()
239 		iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),  in iscsi_tcp_segment_done()
241 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
242 		BUG_ON(segment->size == 0);  in iscsi_tcp_segment_done()
248 		pad = iscsi_padding(segment->total_copied);  in iscsi_tcp_segment_done()
252 			segment->total_size += pad;  in iscsi_tcp_segment_done()
253 			segment->size = pad;  in iscsi_tcp_segment_done()
254 			segment->data = segment->padbuf;  in iscsi_tcp_segment_done()
263 	if (segment->hash) {  in iscsi_tcp_segment_done()
264 		ahash_request_set_crypt(segment->hash, NULL,  in iscsi_tcp_segment_done()
265 					segment->digest, 0);  in iscsi_tcp_segment_done()
266 		crypto_ahash_final(segment->hash);  in iscsi_tcp_segment_done()
267 		iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_segment_done()
268 				 recv ? segment->recv_digest : segment->digest);  in iscsi_tcp_segment_done()
295 		       struct iscsi_segment *segment, const void *ptr,  in iscsi_tcp_segment_recv()  argument
300 	while (!iscsi_tcp_segment_done(tcp_conn, segment, 1, copy)) {  in iscsi_tcp_segment_recv()
307 		copy = min(len - copied, segment->size - segment->copied);  in iscsi_tcp_segment_recv()
309 		memcpy(segment->data + segment->copied, ptr + copied, copy);  in iscsi_tcp_segment_recv()
329 		      struct iscsi_segment *segment)  in iscsi_tcp_dgst_verify()  argument
331 	if (!segment->digest_len)  in iscsi_tcp_dgst_verify()
334 	if (memcmp(segment->recv_digest, segment->digest,  in iscsi_tcp_dgst_verify()
335 		   segment->digest_len)) {  in iscsi_tcp_dgst_verify()
347 __iscsi_segment_init(struct iscsi_segment *segment, size_t size,  in __iscsi_segment_init()  argument
350 	memset(segment, 0, sizeof(*segment));  in __iscsi_segment_init()
351 	segment->total_size = size;  in __iscsi_segment_init()
352 	segment->done = done;  in __iscsi_segment_init()
355 		segment->hash = hash;  in __iscsi_segment_init()
361 iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,  in iscsi_segment_init_linear()  argument
365 	__iscsi_segment_init(segment, size, done, hash);  in iscsi_segment_init_linear()
366 	segment->data = data;  in iscsi_segment_init_linear()
367 	segment->size = size;  in iscsi_segment_init_linear()
372 iscsi_segment_seek_sg(struct iscsi_segment *segment,  in iscsi_segment_seek_sg()  argument
381 	__iscsi_segment_init(segment, size, done, hash);  in iscsi_segment_seek_sg()
384 			iscsi_tcp_segment_init_sg(segment, sg, offset);  in iscsi_segment_seek_sg()
407 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_hdr_recv_prep()
418 			 struct iscsi_segment *segment)  in iscsi_tcp_data_recv_done()  argument
423 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_data_recv_done()
445 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_data_recv_prep()
619 			  struct iscsi_segment *segment)  in iscsi_tcp_process_data_in()  argument
625 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_process_data_in()
714 			rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,  in iscsi_tcp_hdr_dissect()
810 			struct iscsi_segment *segment)  in iscsi_tcp_hdr_recv_done()  argument
820 	if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {  in iscsi_tcp_hdr_recv_done()
830 		segment->total_size += ahslen;  in iscsi_tcp_hdr_recv_done()
831 		segment->size += ahslen;  in iscsi_tcp_hdr_recv_done()
840 		if (segment->digest_len == 0) {  in iscsi_tcp_hdr_recv_done()
846 			iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_hdr_recv_done()
847 							segment->recv_digest);  in iscsi_tcp_hdr_recv_done()
852 				      segment->total_copied - ISCSI_DIGEST_SIZE,  in iscsi_tcp_hdr_recv_done()
853 				      segment->digest);  in iscsi_tcp_hdr_recv_done()
855 		if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_hdr_recv_done()
872 	return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;  in iscsi_tcp_recv_segment_is_hdr()
891 	struct iscsi_segment *segment = &tcp_conn->in.segment;  in iscsi_tcp_recv_skb()  local
911 		segment->total_copied = segment->total_size;  in iscsi_tcp_recv_skb()
927 		BUG_ON(segment->copied >= segment->size);  in iscsi_tcp_recv_skb()
931 		rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);  in iscsi_tcp_recv_skb()
935 		if (segment->total_copied >= segment->total_size) {  in iscsi_tcp_recv_skb()
944 	rc = segment->done(tcp_conn, segment);  in iscsi_tcp_recv_skb()