40 void __ablkcipher_walk_complete(struct ablkcipher_walk *walk)  in __ablkcipher_walk_complete()  argument
44 	list_for_each_entry_safe(p, tmp, &walk->buffers, entry) {  in __ablkcipher_walk_complete()
52 static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk,  in ablkcipher_queue_write()  argument
55 	p->dst = walk->out;  in ablkcipher_queue_write()
56 	list_add_tail(&p->entry, &walk->buffers);  in ablkcipher_queue_write()
69 static inline void ablkcipher_done_slow(struct ablkcipher_walk *walk,  in ablkcipher_done_slow()  argument
73 		unsigned int len_this_page = scatterwalk_pagelen(&walk->out);  in ablkcipher_done_slow()
77 		scatterwalk_advance(&walk->out, n);  in ablkcipher_done_slow()
81 		scatterwalk_start(&walk->out, sg_next(walk->out.sg));  in ablkcipher_done_slow()
85 static inline void ablkcipher_done_fast(struct ablkcipher_walk *walk,  in ablkcipher_done_fast()  argument
88 	scatterwalk_advance(&walk->in, n);  in ablkcipher_done_fast()
89 	scatterwalk_advance(&walk->out, n);  in ablkcipher_done_fast()
93 				struct ablkcipher_walk *walk);
96 			 struct ablkcipher_walk *walk, int err)  in ablkcipher_walk_done()  argument
105 	n = walk->nbytes - err;  in ablkcipher_walk_done()
106 	walk->total -= n;  in ablkcipher_walk_done()
107 	more = (walk->total != 0);  in ablkcipher_walk_done()
109 	if (likely(!(walk->flags & ABLKCIPHER_WALK_SLOW))) {  in ablkcipher_walk_done()
110 		ablkcipher_done_fast(walk, n);  in ablkcipher_walk_done()
117 		ablkcipher_done_slow(walk, n);  in ablkcipher_walk_done()
120 	scatterwalk_done(&walk->in, 0, more);  in ablkcipher_walk_done()
121 	scatterwalk_done(&walk->out, 1, more);  in ablkcipher_walk_done()
125 		return ablkcipher_walk_next(req, walk);  in ablkcipher_walk_done()
129 	walk->nbytes = 0;  in ablkcipher_walk_done()
130 	if (walk->iv != req->info)  in ablkcipher_walk_done()
131 		memcpy(req->info, walk->iv, tfm->crt_ablkcipher.ivsize);  in ablkcipher_walk_done()
132 	kfree(walk->iv_buffer);  in ablkcipher_walk_done()
138 				       struct ablkcipher_walk *walk,  in ablkcipher_next_slow()  argument
154 		return ablkcipher_walk_done(req, walk, -ENOMEM);  in ablkcipher_next_slow()
164 	scatterwalk_copychunks(src, &walk->in, bsize, 0);  in ablkcipher_next_slow()
166 	ablkcipher_queue_write(walk, p);  in ablkcipher_next_slow()
168 	walk->nbytes = bsize;  in ablkcipher_next_slow()
169 	walk->flags |= ABLKCIPHER_WALK_SLOW;  in ablkcipher_next_slow()
177 static inline int ablkcipher_copy_iv(struct ablkcipher_walk *walk,  in ablkcipher_copy_iv()  argument
181 	unsigned bs = walk->blocksize;  in ablkcipher_copy_iv()
189 	walk->iv_buffer = kmalloc(size, GFP_ATOMIC);  in ablkcipher_copy_iv()
190 	if (!walk->iv_buffer)  in ablkcipher_copy_iv()
193 	iv = (u8 *)ALIGN((unsigned long)walk->iv_buffer, alignmask + 1);  in ablkcipher_copy_iv()
198 	walk->iv = memcpy(iv, walk->iv, ivsize);  in ablkcipher_copy_iv()
203 				       struct ablkcipher_walk *walk)  in ablkcipher_next_fast()  argument
205 	walk->src.page = scatterwalk_page(&walk->in);  in ablkcipher_next_fast()
206 	walk->src.offset = offset_in_page(walk->in.offset);  in ablkcipher_next_fast()
207 	walk->dst.page = scatterwalk_page(&walk->out);  in ablkcipher_next_fast()
208 	walk->dst.offset = offset_in_page(walk->out.offset);  in ablkcipher_next_fast()
214 				struct ablkcipher_walk *walk)  in ablkcipher_walk_next()  argument
222 	n = walk->total;  in ablkcipher_walk_next()
225 		return ablkcipher_walk_done(req, walk, -EINVAL);  in ablkcipher_walk_next()
228 	walk->flags &= ~ABLKCIPHER_WALK_SLOW;  in ablkcipher_walk_next()
231 	bsize = min(walk->blocksize, n);  in ablkcipher_walk_next()
232 	n = scatterwalk_clamp(&walk->in, n);  in ablkcipher_walk_next()
233 	n = scatterwalk_clamp(&walk->out, n);  in ablkcipher_walk_next()
236 	    !scatterwalk_aligned(&walk->in, alignmask) ||  in ablkcipher_walk_next()
237 	    !scatterwalk_aligned(&walk->out, alignmask)) {  in ablkcipher_walk_next()
238 		err = ablkcipher_next_slow(req, walk, bsize, alignmask,  in ablkcipher_walk_next()
243 	walk->nbytes = n;  in ablkcipher_walk_next()
245 	return ablkcipher_next_fast(req, walk);  in ablkcipher_walk_next()
249 		walk->src.page = virt_to_page(src);  in ablkcipher_walk_next()
250 		walk->dst.page = virt_to_page(dst);  in ablkcipher_walk_next()
251 		walk->src.offset = ((unsigned long)src & (PAGE_SIZE - 1));  in ablkcipher_walk_next()
252 		walk->dst.offset = ((unsigned long)dst & (PAGE_SIZE - 1));  in ablkcipher_walk_next()
259 				 struct ablkcipher_walk *walk)  in ablkcipher_walk_first()  argument
268 	walk->iv = req->info;  in ablkcipher_walk_first()
269 	walk->nbytes = walk->total;  in ablkcipher_walk_first()
270 	if (unlikely(!walk->total))  in ablkcipher_walk_first()
273 	walk->iv_buffer = NULL;  in ablkcipher_walk_first()
274 	if (unlikely(((unsigned long)walk->iv & alignmask))) {  in ablkcipher_walk_first()
275 		int err = ablkcipher_copy_iv(walk, tfm, alignmask);  in ablkcipher_walk_first()
281 	scatterwalk_start(&walk->in, walk->in.sg);  in ablkcipher_walk_first()
282 	scatterwalk_start(&walk->out, walk->out.sg);  in ablkcipher_walk_first()
284 	return ablkcipher_walk_next(req, walk);  in ablkcipher_walk_first()
288 			 struct ablkcipher_walk *walk)  in ablkcipher_walk_phys()  argument
290 	walk->blocksize = crypto_tfm_alg_blocksize(req->base.tfm);  in ablkcipher_walk_phys()
291 	return ablkcipher_walk_first(req, walk);  in ablkcipher_walk_phys()