3  * Handle async block request by crypto hardware engine.
13 #include <crypto/engine.h>
21  * @engine: the hardware engine
25 static void crypto_finalize_request(struct crypto_engine *engine,  in crypto_finalize_request()  argument
38 	if (!engine->retry_support) {  in crypto_finalize_request()
39 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_finalize_request()
40 		if (engine->cur_req == req) {  in crypto_finalize_request()
42 			engine->cur_req = NULL;  in crypto_finalize_request()
44 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_finalize_request()
47 	if (finalize_req || engine->retry_support) {  in crypto_finalize_request()
51 			ret = enginectx->op.unprepare_request(engine, req);  in crypto_finalize_request()
53 				dev_err(engine->dev, "failed to unprepare request\n");  in crypto_finalize_request()
59 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_finalize_request()
63  * crypto_pump_requests - dequeue one request from engine queue to process
64  * @engine: the hardware engine
67  * This function checks if there is any request in the engine queue that
71 static void crypto_pump_requests(struct crypto_engine *engine,  in crypto_pump_requests()  argument
80 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
83 	if (!engine->retry_support && engine->cur_req)  in crypto_pump_requests()
87 	if (engine->idling) {  in crypto_pump_requests()
88 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_pump_requests()
92 	/* Check if the engine queue is idle */  in crypto_pump_requests()
93 	if (!crypto_queue_len(&engine->queue) || !engine->running) {  in crypto_pump_requests()
94 		if (!engine->busy)  in crypto_pump_requests()
99 			kthread_queue_work(engine->kworker,  in crypto_pump_requests()
100 					   &engine->pump_requests);  in crypto_pump_requests()
104 		engine->busy = false;  in crypto_pump_requests()
105 		engine->idling = true;  in crypto_pump_requests()
106 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
108 		if (engine->unprepare_crypt_hardware &&  in crypto_pump_requests()
109 		    engine->unprepare_crypt_hardware(engine))  in crypto_pump_requests()
110 			dev_err(engine->dev, "failed to unprepare crypt hardware\n");  in crypto_pump_requests()
112 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
113 		engine->idling = false;  in crypto_pump_requests()
118 	/* Get the fist request from the engine queue to handle */  in crypto_pump_requests()
119 	backlog = crypto_get_backlog(&engine->queue);  in crypto_pump_requests()
120 	async_req = crypto_dequeue_request(&engine->queue);  in crypto_pump_requests()
129 	if (!engine->retry_support)  in crypto_pump_requests()
130 		engine->cur_req = async_req;  in crypto_pump_requests()
135 	if (engine->busy)  in crypto_pump_requests()
138 		engine->busy = true;  in crypto_pump_requests()
140 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
143 	if (!was_busy && engine->prepare_crypt_hardware) {  in crypto_pump_requests()
144 		ret = engine->prepare_crypt_hardware(engine);  in crypto_pump_requests()
146 			dev_err(engine->dev, "failed to prepare crypt hardware\n");  in crypto_pump_requests()
154 		ret = enginectx->op.prepare_request(engine, async_req);  in crypto_pump_requests()
156 			dev_err(engine->dev, "failed to prepare request: %d\n",  in crypto_pump_requests()
162 		dev_err(engine->dev, "failed to do request\n");  in crypto_pump_requests()
167 	ret = enginectx->op.do_one_request(engine, async_req);  in crypto_pump_requests()
176 		if (!engine->retry_support ||  in crypto_pump_requests()
178 			dev_err(engine->dev,  in crypto_pump_requests()
186 		 * enqueue it back into crypto-engine queue.  in crypto_pump_requests()
189 			ret = enginectx->op.unprepare_request(engine,  in crypto_pump_requests()
192 				dev_err(engine->dev,  in crypto_pump_requests()
195 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
198 		 * back in front of crypto-engine queue, to keep the order  in crypto_pump_requests()
201 		crypto_enqueue_request_head(&engine->queue, async_req);  in crypto_pump_requests()
203 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_pump_requests()
211 		ret = enginectx->op.unprepare_request(engine, async_req);  in crypto_pump_requests()
213 			dev_err(engine->dev, "failed to unprepare request\n");  in crypto_pump_requests()
220 	/* If retry mechanism is supported, send new requests to engine */  in crypto_pump_requests()
221 	if (engine->retry_support) {  in crypto_pump_requests()
222 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
228 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
234 	if (engine->do_batch_requests) {  in crypto_pump_requests()
235 		ret = engine->do_batch_requests(engine);  in crypto_pump_requests()
237 			dev_err(engine->dev, "failed to do batch requests: %d\n",  in crypto_pump_requests()
246 	struct crypto_engine *engine =  in crypto_pump_work()  local
249 	crypto_pump_requests(engine, true);  in crypto_pump_work()
253  * crypto_transfer_request - transfer the new request into the engine queue
254  * @engine: the hardware engine
255  * @req: the request need to be listed into the engine queue
258 static int crypto_transfer_request(struct crypto_engine *engine,  in crypto_transfer_request()  argument
265 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_transfer_request()
267 	if (!engine->running) {  in crypto_transfer_request()
268 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
272 	ret = crypto_enqueue_request(&engine->queue, req);  in crypto_transfer_request()
274 	if (!engine->busy && need_pump)  in crypto_transfer_request()
275 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_transfer_request()
277 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
283  * into the engine queue
284  * @engine: the hardware engine
285  * @req: the request need to be listed into the engine queue
287 static int crypto_transfer_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_request_to_engine()  argument
290 	return crypto_transfer_request(engine, req, true);  in crypto_transfer_request_to_engine()
295  * to list into the engine queue
296  * @engine: the hardware engine
297  * @req: the request need to be listed into the engine queue
299 int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_aead_request_to_engine()  argument
302 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_aead_request_to_engine()
308  * to list into the engine queue
309  * @engine: the hardware engine
310  * @req: the request need to be listed into the engine queue
312 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_akcipher_request_to_engine()  argument
315 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_akcipher_request_to_engine()
321  * to list into the engine queue
322  * @engine: the hardware engine
323  * @req: the request need to be listed into the engine queue
325 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_hash_request_to_engine()  argument
328 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_hash_request_to_engine()
334  * into the engine queue
335  * @engine: the hardware engine
336  * @req: the request need to be listed into the engine queue
338 int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_kpp_request_to_engine()  argument
341 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_kpp_request_to_engine()
347  * to list into the engine queue
348  * @engine: the hardware engine
349  * @req: the request need to be listed into the engine queue
351 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_skcipher_request_to_engine()  argument
354 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_skcipher_request_to_engine()
361  * @engine: the hardware engine
365 void crypto_finalize_aead_request(struct crypto_engine *engine,  in crypto_finalize_aead_request()  argument
368 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_aead_request()
375  * @engine: the hardware engine
379 void crypto_finalize_akcipher_request(struct crypto_engine *engine,  in crypto_finalize_akcipher_request()  argument
382 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_akcipher_request()
389  * @engine: the hardware engine
393 void crypto_finalize_hash_request(struct crypto_engine *engine,  in crypto_finalize_hash_request()  argument
396 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_hash_request()
402  * @engine: the hardware engine
406 void crypto_finalize_kpp_request(struct crypto_engine *engine,  in crypto_finalize_kpp_request()  argument
409 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_kpp_request()
416  * @engine: the hardware engine
420 void crypto_finalize_skcipher_request(struct crypto_engine *engine,  in crypto_finalize_skcipher_request()  argument
423 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_skcipher_request()
428  * crypto_engine_start - start the hardware engine
429  * @engine: the hardware engine need to be started
433 int crypto_engine_start(struct crypto_engine *engine)  in crypto_engine_start()  argument
437 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_start()
439 	if (engine->running || engine->busy) {  in crypto_engine_start()
440 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
444 	engine->running = true;  in crypto_engine_start()
445 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
447 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_engine_start()
454  * crypto_engine_stop - stop the hardware engine
455  * @engine: the hardware engine need to be stopped
459 int crypto_engine_stop(struct crypto_engine *engine)  in crypto_engine_stop()  argument
465 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
468 	 * If the engine queue is not empty or the engine is on busy state,  in crypto_engine_stop()
469 	 * we need to wait for a while to pump the requests of engine queue.  in crypto_engine_stop()
471 	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {  in crypto_engine_stop()
472 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
474 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
477 	if (crypto_queue_len(&engine->queue) || engine->busy)  in crypto_engine_stop()
480 		engine->running = false;  in crypto_engine_stop()
482 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
485 		dev_warn(engine->dev, "could not stop engine\n");  in crypto_engine_stop()
492  * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
494  * crypto-engine queue.
495  * @dev: the device attached with one hardware engine
500  *                callback(struct crypto_engine *engine)
502  *                @engine: the crypto engine structure.
504  * @qlen: maximum size of the crypto-engine queue
507  * Return: the crypto engine structure on success, else NULL.
511 						       int (*cbk_do_batch)(struct crypto_engine *engine),  in crypto_engine_alloc_init_and_set()  argument
514 	struct crypto_engine *engine;  in crypto_engine_alloc_init_and_set()  local
519 	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);  in crypto_engine_alloc_init_and_set()
520 	if (!engine)  in crypto_engine_alloc_init_and_set()
523 	engine->dev = dev;  in crypto_engine_alloc_init_and_set()
524 	engine->rt = rt;  in crypto_engine_alloc_init_and_set()
525 	engine->running = false;  in crypto_engine_alloc_init_and_set()
526 	engine->busy = false;  in crypto_engine_alloc_init_and_set()
527 	engine->idling = false;  in crypto_engine_alloc_init_and_set()
528 	engine->retry_support = retry_support;  in crypto_engine_alloc_init_and_set()
529 	engine->priv_data = dev;  in crypto_engine_alloc_init_and_set()
534 	engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;  in crypto_engine_alloc_init_and_set()
536 	snprintf(engine->name, sizeof(engine->name),  in crypto_engine_alloc_init_and_set()
537 		 "%s-engine", dev_name(dev));  in crypto_engine_alloc_init_and_set()
539 	crypto_init_queue(&engine->queue, qlen);  in crypto_engine_alloc_init_and_set()
540 	spin_lock_init(&engine->queue_lock);  in crypto_engine_alloc_init_and_set()
542 	engine->kworker = kthread_create_worker(0, "%s", engine->name);  in crypto_engine_alloc_init_and_set()
543 	if (IS_ERR(engine->kworker)) {  in crypto_engine_alloc_init_and_set()
547 	kthread_init_work(&engine->pump_requests, crypto_pump_work);  in crypto_engine_alloc_init_and_set()
549 	if (engine->rt) {  in crypto_engine_alloc_init_and_set()
551 		sched_set_fifo(engine->kworker->task);  in crypto_engine_alloc_init_and_set()
554 	return engine;  in crypto_engine_alloc_init_and_set()
559  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
561  * @dev: the device attached with one hardware engine
565  * Return: the crypto engine structure on success, else NULL.
575  * crypto_engine_exit - free the resources of hardware engine when exit
576  * @engine: the hardware engine need to be freed
580 int crypto_engine_exit(struct crypto_engine *engine)  in crypto_engine_exit()  argument
584 	ret = crypto_engine_stop(engine);  in crypto_engine_exit()
588 	kthread_destroy_worker(engine->kworker);  in crypto_engine_exit()
595 MODULE_DESCRIPTION("Crypto hardware engine framework");