25 static void crypto_finalize_request(struct crypto_engine *engine,  in crypto_finalize_request()  argument
38 	if (!engine->retry_support) {  in crypto_finalize_request()
39 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_finalize_request()
40 		if (engine->cur_req == req) {  in crypto_finalize_request()
42 			engine->cur_req = NULL;  in crypto_finalize_request()
44 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_finalize_request()
47 	if (finalize_req || engine->retry_support) {  in crypto_finalize_request()
51 			ret = enginectx->op.unprepare_request(engine, req);  in crypto_finalize_request()
53 				dev_err(engine->dev, "failed to unprepare request\n");  in crypto_finalize_request()
59 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_finalize_request()
71 static void crypto_pump_requests(struct crypto_engine *engine,  in crypto_pump_requests()  argument
80 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
83 	if (!engine->retry_support && engine->cur_req)  in crypto_pump_requests()
87 	if (engine->idling) {  in crypto_pump_requests()
88 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_pump_requests()
93 	if (!crypto_queue_len(&engine->queue) || !engine->running) {  in crypto_pump_requests()
94 		if (!engine->busy)  in crypto_pump_requests()
99 			kthread_queue_work(engine->kworker,  in crypto_pump_requests()
100 					   &engine->pump_requests);  in crypto_pump_requests()
104 		engine->busy = false;  in crypto_pump_requests()
105 		engine->idling = true;  in crypto_pump_requests()
106 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
108 		if (engine->unprepare_crypt_hardware &&  in crypto_pump_requests()
109 		    engine->unprepare_crypt_hardware(engine))  in crypto_pump_requests()
110 			dev_err(engine->dev, "failed to unprepare crypt hardware\n");  in crypto_pump_requests()
112 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
113 		engine->idling = false;  in crypto_pump_requests()
119 	backlog = crypto_get_backlog(&engine->queue);  in crypto_pump_requests()
120 	async_req = crypto_dequeue_request(&engine->queue);  in crypto_pump_requests()
129 	if (!engine->retry_support)  in crypto_pump_requests()
130 		engine->cur_req = async_req;  in crypto_pump_requests()
135 	if (engine->busy)  in crypto_pump_requests()
138 		engine->busy = true;  in crypto_pump_requests()
140 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
143 	if (!was_busy && engine->prepare_crypt_hardware) {  in crypto_pump_requests()
144 		ret = engine->prepare_crypt_hardware(engine);  in crypto_pump_requests()
146 			dev_err(engine->dev, "failed to prepare crypt hardware\n");  in crypto_pump_requests()
154 		ret = enginectx->op.prepare_request(engine, async_req);  in crypto_pump_requests()
156 			dev_err(engine->dev, "failed to prepare request: %d\n",  in crypto_pump_requests()
162 		dev_err(engine->dev, "failed to do request\n");  in crypto_pump_requests()
167 	ret = enginectx->op.do_one_request(engine, async_req);  in crypto_pump_requests()
176 		if (!engine->retry_support ||  in crypto_pump_requests()
178 			dev_err(engine->dev,  in crypto_pump_requests()
189 			ret = enginectx->op.unprepare_request(engine,  in crypto_pump_requests()
192 				dev_err(engine->dev,  in crypto_pump_requests()
195 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
201 		crypto_enqueue_request_head(&engine->queue, async_req);  in crypto_pump_requests()
203 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_pump_requests()
211 		ret = enginectx->op.unprepare_request(engine, async_req);  in crypto_pump_requests()
213 			dev_err(engine->dev, "failed to unprepare request\n");  in crypto_pump_requests()
221 	if (engine->retry_support) {  in crypto_pump_requests()
222 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
228 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
234 	if (engine->do_batch_requests) {  in crypto_pump_requests()
235 		ret = engine->do_batch_requests(engine);  in crypto_pump_requests()
237 			dev_err(engine->dev, "failed to do batch requests: %d\n",  in crypto_pump_requests()
246 	struct crypto_engine *engine =  in crypto_pump_work()  local
249 	crypto_pump_requests(engine, true);  in crypto_pump_work()
258 static int crypto_transfer_request(struct crypto_engine *engine,  in crypto_transfer_request()  argument
265 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_transfer_request()
267 	if (!engine->running) {  in crypto_transfer_request()
268 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
272 	ret = crypto_enqueue_request(&engine->queue, req);  in crypto_transfer_request()
274 	if (!engine->busy && need_pump)  in crypto_transfer_request()
275 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_transfer_request()
277 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
287 static int crypto_transfer_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_request_to_engine()  argument
290 	return crypto_transfer_request(engine, req, true);  in crypto_transfer_request_to_engine()
299 int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_aead_request_to_engine()  argument
302 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_aead_request_to_engine()
312 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_akcipher_request_to_engine()  argument
315 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_akcipher_request_to_engine()
325 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_hash_request_to_engine()  argument
328 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_hash_request_to_engine()
338 int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_kpp_request_to_engine()  argument
341 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_kpp_request_to_engine()
351 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_skcipher_request_to_engine()  argument
354 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_skcipher_request_to_engine()
365 void crypto_finalize_aead_request(struct crypto_engine *engine,  in crypto_finalize_aead_request()  argument
368 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_aead_request()
379 void crypto_finalize_akcipher_request(struct crypto_engine *engine,  in crypto_finalize_akcipher_request()  argument
382 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_akcipher_request()
393 void crypto_finalize_hash_request(struct crypto_engine *engine,  in crypto_finalize_hash_request()  argument
396 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_hash_request()
406 void crypto_finalize_kpp_request(struct crypto_engine *engine,  in crypto_finalize_kpp_request()  argument
409 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_kpp_request()
420 void crypto_finalize_skcipher_request(struct crypto_engine *engine,  in crypto_finalize_skcipher_request()  argument
423 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_skcipher_request()
433 int crypto_engine_start(struct crypto_engine *engine)  in crypto_engine_start()  argument
437 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_start()
439 	if (engine->running || engine->busy) {  in crypto_engine_start()
440 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
444 	engine->running = true;  in crypto_engine_start()
445 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
447 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_engine_start()
459 int crypto_engine_stop(struct crypto_engine *engine)  in crypto_engine_stop()  argument
465 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
471 	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {  in crypto_engine_stop()
472 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
474 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
477 	if (crypto_queue_len(&engine->queue) || engine->busy)  in crypto_engine_stop()
480 		engine->running = false;  in crypto_engine_stop()
482 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
485 		dev_warn(engine->dev, "could not stop engine\n");  in crypto_engine_stop()
511 						       int (*cbk_do_batch)(struct crypto_engine *engine),  in crypto_engine_alloc_init_and_set()  argument
514 	struct crypto_engine *engine;  in crypto_engine_alloc_init_and_set()  local
519 	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);  in crypto_engine_alloc_init_and_set()
520 	if (!engine)  in crypto_engine_alloc_init_and_set()
523 	engine->dev = dev;  in crypto_engine_alloc_init_and_set()
524 	engine->rt = rt;  in crypto_engine_alloc_init_and_set()
525 	engine->running = false;  in crypto_engine_alloc_init_and_set()
526 	engine->busy = false;  in crypto_engine_alloc_init_and_set()
527 	engine->idling = false;  in crypto_engine_alloc_init_and_set()
528 	engine->retry_support = retry_support;  in crypto_engine_alloc_init_and_set()
529 	engine->priv_data = dev;  in crypto_engine_alloc_init_and_set()
534 	engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;  in crypto_engine_alloc_init_and_set()
536 	snprintf(engine->name, sizeof(engine->name),  in crypto_engine_alloc_init_and_set()
539 	crypto_init_queue(&engine->queue, qlen);  in crypto_engine_alloc_init_and_set()
540 	spin_lock_init(&engine->queue_lock);  in crypto_engine_alloc_init_and_set()
542 	engine->kworker = kthread_create_worker(0, "%s", engine->name);  in crypto_engine_alloc_init_and_set()
543 	if (IS_ERR(engine->kworker)) {  in crypto_engine_alloc_init_and_set()
547 	kthread_init_work(&engine->pump_requests, crypto_pump_work);  in crypto_engine_alloc_init_and_set()
549 	if (engine->rt) {  in crypto_engine_alloc_init_and_set()
551 		sched_set_fifo(engine->kworker->task);  in crypto_engine_alloc_init_and_set()
554 	return engine;  in crypto_engine_alloc_init_and_set()
580 int crypto_engine_exit(struct crypto_engine *engine)  in crypto_engine_exit()  argument
584 	ret = crypto_engine_stop(engine);  in crypto_engine_exit()
588 	kthread_destroy_worker(engine->kworker);  in crypto_engine_exit()