29 static void crypto_finalize_request(struct crypto_engine *engine,  in crypto_finalize_request()  argument
37 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_finalize_request()
38 	if (engine->cur_req == req)  in crypto_finalize_request()
40 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_finalize_request()
44 		if (engine->cur_req_prepared &&  in crypto_finalize_request()
46 			ret = enginectx->op.unprepare_request(engine, req);  in crypto_finalize_request()
48 				dev_err(engine->dev, "failed to unprepare request\n");  in crypto_finalize_request()
50 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_finalize_request()
51 		engine->cur_req = NULL;  in crypto_finalize_request()
52 		engine->cur_req_prepared = false;  in crypto_finalize_request()
53 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_finalize_request()
58 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_finalize_request()
70 static void crypto_pump_requests(struct crypto_engine *engine,  in crypto_pump_requests()  argument
79 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
82 	if (engine->cur_req)  in crypto_pump_requests()
86 	if (engine->idling) {  in crypto_pump_requests()
87 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_pump_requests()
92 	if (!crypto_queue_len(&engine->queue) || !engine->running) {  in crypto_pump_requests()
93 		if (!engine->busy)  in crypto_pump_requests()
98 			kthread_queue_work(engine->kworker,  in crypto_pump_requests()
99 					   &engine->pump_requests);  in crypto_pump_requests()
103 		engine->busy = false;  in crypto_pump_requests()
104 		engine->idling = true;  in crypto_pump_requests()
105 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
107 		if (engine->unprepare_crypt_hardware &&  in crypto_pump_requests()
108 		    engine->unprepare_crypt_hardware(engine))  in crypto_pump_requests()
109 			dev_err(engine->dev, "failed to unprepare crypt hardware\n");  in crypto_pump_requests()
111 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
112 		engine->idling = false;  in crypto_pump_requests()
117 	backlog = crypto_get_backlog(&engine->queue);  in crypto_pump_requests()
118 	async_req = crypto_dequeue_request(&engine->queue);  in crypto_pump_requests()
122 	engine->cur_req = async_req;  in crypto_pump_requests()
126 	if (engine->busy)  in crypto_pump_requests()
129 		engine->busy = true;  in crypto_pump_requests()
131 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
134 	if (!was_busy && engine->prepare_crypt_hardware) {  in crypto_pump_requests()
135 		ret = engine->prepare_crypt_hardware(engine);  in crypto_pump_requests()
137 			dev_err(engine->dev, "failed to prepare crypt hardware\n");  in crypto_pump_requests()
145 		ret = enginectx->op.prepare_request(engine, async_req);  in crypto_pump_requests()
147 			dev_err(engine->dev, "failed to prepare request: %d\n",  in crypto_pump_requests()
151 		engine->cur_req_prepared = true;  in crypto_pump_requests()
154 		dev_err(engine->dev, "failed to do request\n");  in crypto_pump_requests()
158 	ret = enginectx->op.do_one_request(engine, async_req);  in crypto_pump_requests()
160 		dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret);  in crypto_pump_requests()
166 	crypto_finalize_request(engine, async_req, ret);  in crypto_pump_requests()
170 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
175 	struct crypto_engine *engine =  in crypto_pump_work()  local
178 	crypto_pump_requests(engine, true);  in crypto_pump_work()
186 static int crypto_transfer_request(struct crypto_engine *engine,  in crypto_transfer_request()  argument
193 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_transfer_request()
195 	if (!engine->running) {  in crypto_transfer_request()
196 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
200 	ret = crypto_enqueue_request(&engine->queue, req);  in crypto_transfer_request()
202 	if (!engine->busy && need_pump)  in crypto_transfer_request()
203 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_transfer_request()
205 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
215 static int crypto_transfer_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_request_to_engine()  argument
218 	return crypto_transfer_request(engine, req, true);  in crypto_transfer_request_to_engine()
228 int crypto_transfer_ablkcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_ablkcipher_request_to_engine()  argument
231 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_ablkcipher_request_to_engine()
241 int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_aead_request_to_engine()  argument
244 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_aead_request_to_engine()
254 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_akcipher_request_to_engine()  argument
257 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_akcipher_request_to_engine()
267 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_hash_request_to_engine()  argument
270 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_hash_request_to_engine()
280 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_skcipher_request_to_engine()  argument
283 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_skcipher_request_to_engine()
295 void crypto_finalize_ablkcipher_request(struct crypto_engine *engine,  in crypto_finalize_ablkcipher_request()  argument
298 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_ablkcipher_request()
309 void crypto_finalize_aead_request(struct crypto_engine *engine,  in crypto_finalize_aead_request()  argument
312 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_aead_request()
323 void crypto_finalize_akcipher_request(struct crypto_engine *engine,  in crypto_finalize_akcipher_request()  argument
326 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_akcipher_request()
337 void crypto_finalize_hash_request(struct crypto_engine *engine,  in crypto_finalize_hash_request()  argument
340 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_hash_request()
351 void crypto_finalize_skcipher_request(struct crypto_engine *engine,  in crypto_finalize_skcipher_request()  argument
354 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_skcipher_request()
364 int crypto_engine_start(struct crypto_engine *engine)  in crypto_engine_start()  argument
368 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_start()
370 	if (engine->running || engine->busy) {  in crypto_engine_start()
371 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
375 	engine->running = true;  in crypto_engine_start()
376 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
378 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_engine_start()
390 int crypto_engine_stop(struct crypto_engine *engine)  in crypto_engine_stop()  argument
396 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
402 	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {  in crypto_engine_stop()
403 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
405 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
408 	if (crypto_queue_len(&engine->queue) || engine->busy)  in crypto_engine_stop()
411 		engine->running = false;  in crypto_engine_stop()
413 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
416 		dev_warn(engine->dev, "could not stop engine\n");  in crypto_engine_stop()
434 	struct crypto_engine *engine;  in crypto_engine_alloc_init()  local
439 	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);  in crypto_engine_alloc_init()
440 	if (!engine)  in crypto_engine_alloc_init()
443 	engine->dev = dev;  in crypto_engine_alloc_init()
444 	engine->rt = rt;  in crypto_engine_alloc_init()
445 	engine->running = false;  in crypto_engine_alloc_init()
446 	engine->busy = false;  in crypto_engine_alloc_init()
447 	engine->idling = false;  in crypto_engine_alloc_init()
448 	engine->cur_req_prepared = false;  in crypto_engine_alloc_init()
449 	engine->priv_data = dev;  in crypto_engine_alloc_init()
450 	snprintf(engine->name, sizeof(engine->name),  in crypto_engine_alloc_init()
453 	crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);  in crypto_engine_alloc_init()
454 	spin_lock_init(&engine->queue_lock);  in crypto_engine_alloc_init()
456 	engine->kworker = kthread_create_worker(0, "%s", engine->name);  in crypto_engine_alloc_init()
457 	if (IS_ERR(engine->kworker)) {  in crypto_engine_alloc_init()
461 	kthread_init_work(&engine->pump_requests, crypto_pump_work);  in crypto_engine_alloc_init()
463 	if (engine->rt) {  in crypto_engine_alloc_init()
465 		sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);  in crypto_engine_alloc_init()
468 	return engine;  in crypto_engine_alloc_init()
478 int crypto_engine_exit(struct crypto_engine *engine)  in crypto_engine_exit()  argument
482 	ret = crypto_engine_stop(engine);  in crypto_engine_exit()
486 	kthread_destroy_worker(engine->kworker);  in crypto_engine_exit()