75 	struct atmel_tdes_dev *dd;  member
181 static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset)  in atmel_tdes_read()  argument
183 	return readl_relaxed(dd->io_base + offset);  in atmel_tdes_read()
186 static inline void atmel_tdes_write(struct atmel_tdes_dev *dd,  in atmel_tdes_write()  argument
189 	writel_relaxed(value, dd->io_base + offset);  in atmel_tdes_write()
192 static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset,  in atmel_tdes_write_n()  argument
196 		atmel_tdes_write(dd, offset, *value);  in atmel_tdes_write_n()
205 	if (!ctx->dd) {  in atmel_tdes_find_dev()
210 		ctx->dd = tdes_dd;  in atmel_tdes_find_dev()
212 		tdes_dd = ctx->dd;  in atmel_tdes_find_dev()
219 static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd)  in atmel_tdes_hw_init()  argument
223 	err = clk_prepare_enable(dd->iclk);  in atmel_tdes_hw_init()
227 	if (!(dd->flags & TDES_FLAGS_INIT)) {  in atmel_tdes_hw_init()
228 		atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST);  in atmel_tdes_hw_init()
229 		dd->flags |= TDES_FLAGS_INIT;  in atmel_tdes_hw_init()
235 static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd)  in atmel_tdes_get_version()  argument
237 	return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff;  in atmel_tdes_get_version()
240 static int atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd)  in atmel_tdes_hw_version_init()  argument
244 	err = atmel_tdes_hw_init(dd);  in atmel_tdes_hw_version_init()
248 	dd->hw_version = atmel_tdes_get_version(dd);  in atmel_tdes_hw_version_init()
250 	dev_info(dd->dev,  in atmel_tdes_hw_version_init()
251 			"version: 0x%x\n", dd->hw_version);  in atmel_tdes_hw_version_init()
253 	clk_disable_unprepare(dd->iclk);  in atmel_tdes_hw_version_init()
260 	struct atmel_tdes_dev *dd = data;  in atmel_tdes_dma_callback()  local
263 	tasklet_schedule(&dd->done_task);  in atmel_tdes_dma_callback()
266 static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd)  in atmel_tdes_write_ctrl()  argument
271 	err = atmel_tdes_hw_init(dd);  in atmel_tdes_write_ctrl()
276 	if (!dd->caps.has_dma)  in atmel_tdes_write_ctrl()
277 		atmel_tdes_write(dd, TDES_PTCR,  in atmel_tdes_write_ctrl()
281 	if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) {  in atmel_tdes_write_ctrl()
284 	} else if (dd->ctx->keylen > DES_KEY_SIZE) {  in atmel_tdes_write_ctrl()
291 	valmr |= dd->flags & TDES_FLAGS_MODE_MASK;  in atmel_tdes_write_ctrl()
293 	atmel_tdes_write(dd, TDES_MR, valmr);  in atmel_tdes_write_ctrl()
295 	atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key,  in atmel_tdes_write_ctrl()
296 						dd->ctx->keylen >> 2);  in atmel_tdes_write_ctrl()
298 	if (dd->req->iv && (valmr & TDES_MR_OPMOD_MASK) != TDES_MR_OPMOD_ECB)  in atmel_tdes_write_ctrl()
299 		atmel_tdes_write_n(dd, TDES_IV1R, (void *)dd->req->iv, 2);  in atmel_tdes_write_ctrl()
304 static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)  in atmel_tdes_crypt_pdc_stop()  argument
309 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);  in atmel_tdes_crypt_pdc_stop()
311 	if (dd->flags & TDES_FLAGS_FAST) {  in atmel_tdes_crypt_pdc_stop()
312 		dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);  in atmel_tdes_crypt_pdc_stop()
313 		dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_pdc_stop()
315 		dma_sync_single_for_device(dd->dev, dd->dma_addr_out,  in atmel_tdes_crypt_pdc_stop()
316 					   dd->dma_size, DMA_FROM_DEVICE);  in atmel_tdes_crypt_pdc_stop()
319 		count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,  in atmel_tdes_crypt_pdc_stop()
320 				dd->buf_out, dd->buflen, dd->dma_size, 1);  in atmel_tdes_crypt_pdc_stop()
321 		if (count != dd->dma_size) {  in atmel_tdes_crypt_pdc_stop()
330 static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)  in atmel_tdes_buff_init()  argument
334 	dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0);  in atmel_tdes_buff_init()
335 	dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0);  in atmel_tdes_buff_init()
336 	dd->buflen = PAGE_SIZE;  in atmel_tdes_buff_init()
337 	dd->buflen &= ~(DES_BLOCK_SIZE - 1);  in atmel_tdes_buff_init()
339 	if (!dd->buf_in || !dd->buf_out) {  in atmel_tdes_buff_init()
340 		dev_err(dd->dev, "unable to alloc pages.\n");  in atmel_tdes_buff_init()
345 	dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in,  in atmel_tdes_buff_init()
346 					dd->buflen, DMA_TO_DEVICE);  in atmel_tdes_buff_init()
347 	if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {  in atmel_tdes_buff_init()
348 		dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen);  in atmel_tdes_buff_init()
353 	dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out,  in atmel_tdes_buff_init()
354 					dd->buflen, DMA_FROM_DEVICE);  in atmel_tdes_buff_init()
355 	if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {  in atmel_tdes_buff_init()
356 		dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen);  in atmel_tdes_buff_init()
364 	dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,  in atmel_tdes_buff_init()
368 	free_page((unsigned long)dd->buf_out);  in atmel_tdes_buff_init()
369 	free_page((unsigned long)dd->buf_in);  in atmel_tdes_buff_init()
375 static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd)  in atmel_tdes_buff_cleanup()  argument
377 	dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,  in atmel_tdes_buff_cleanup()
379 	dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,  in atmel_tdes_buff_cleanup()
381 	free_page((unsigned long)dd->buf_out);  in atmel_tdes_buff_cleanup()
382 	free_page((unsigned long)dd->buf_in);  in atmel_tdes_buff_cleanup()
385 static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd,  in atmel_tdes_crypt_pdc()  argument
389 	struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(dd->req);  in atmel_tdes_crypt_pdc()
392 	dd->dma_size = length;  in atmel_tdes_crypt_pdc()
394 	if (!(dd->flags & TDES_FLAGS_FAST)) {  in atmel_tdes_crypt_pdc()
395 		dma_sync_single_for_device(dd->dev, dma_addr_in, length,  in atmel_tdes_crypt_pdc()
413 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);  in atmel_tdes_crypt_pdc()
414 	atmel_tdes_write(dd, TDES_TPR, dma_addr_in);  in atmel_tdes_crypt_pdc()
415 	atmel_tdes_write(dd, TDES_TCR, len32);  in atmel_tdes_crypt_pdc()
416 	atmel_tdes_write(dd, TDES_RPR, dma_addr_out);  in atmel_tdes_crypt_pdc()
417 	atmel_tdes_write(dd, TDES_RCR, len32);  in atmel_tdes_crypt_pdc()
420 	atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX);  in atmel_tdes_crypt_pdc()
423 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN);  in atmel_tdes_crypt_pdc()
428 static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd,  in atmel_tdes_crypt_dma()  argument
432 	struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(dd->req);  in atmel_tdes_crypt_dma()
437 	dd->dma_size = length;  in atmel_tdes_crypt_dma()
439 	if (!(dd->flags & TDES_FLAGS_FAST)) {  in atmel_tdes_crypt_dma()
440 		dma_sync_single_for_device(dd->dev, dma_addr_in, length,  in atmel_tdes_crypt_dma()
458 	dd->dma_lch_in.dma_conf.dst_addr_width = addr_width;  in atmel_tdes_crypt_dma()
459 	dd->dma_lch_out.dma_conf.src_addr_width = addr_width;  in atmel_tdes_crypt_dma()
461 	dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);  in atmel_tdes_crypt_dma()
462 	dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf);  in atmel_tdes_crypt_dma()
464 	dd->flags |= TDES_FLAGS_DMA;  in atmel_tdes_crypt_dma()
474 	in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0],  in atmel_tdes_crypt_dma()
480 	out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1],  in atmel_tdes_crypt_dma()
487 	out_desc->callback_param = dd;  in atmel_tdes_crypt_dma()
490 	dma_async_issue_pending(dd->dma_lch_out.chan);  in atmel_tdes_crypt_dma()
493 	dma_async_issue_pending(dd->dma_lch_in.chan);  in atmel_tdes_crypt_dma()
498 static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)  in atmel_tdes_crypt_start()  argument
504 	if ((!dd->in_offset) && (!dd->out_offset)) {  in atmel_tdes_crypt_start()
506 		in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) &&  in atmel_tdes_crypt_start()
507 			IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size);  in atmel_tdes_crypt_start()
508 		out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) &&  in atmel_tdes_crypt_start()
509 			IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size);  in atmel_tdes_crypt_start()
512 		if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg))  in atmel_tdes_crypt_start()
518 		count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg));  in atmel_tdes_crypt_start()
519 		count = min_t(size_t, count, sg_dma_len(dd->out_sg));  in atmel_tdes_crypt_start()
521 		err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_start()
523 			dev_err(dd->dev, "dma_map_sg() error\n");  in atmel_tdes_crypt_start()
527 		err = dma_map_sg(dd->dev, dd->out_sg, 1,  in atmel_tdes_crypt_start()
530 			dev_err(dd->dev, "dma_map_sg() error\n");  in atmel_tdes_crypt_start()
531 			dma_unmap_sg(dd->dev, dd->in_sg, 1,  in atmel_tdes_crypt_start()
536 		addr_in = sg_dma_address(dd->in_sg);  in atmel_tdes_crypt_start()
537 		addr_out = sg_dma_address(dd->out_sg);  in atmel_tdes_crypt_start()
539 		dd->flags |= TDES_FLAGS_FAST;  in atmel_tdes_crypt_start()
543 		count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset,  in atmel_tdes_crypt_start()
544 				dd->buf_in, dd->buflen, dd->total, 0);  in atmel_tdes_crypt_start()
546 		addr_in = dd->dma_addr_in;  in atmel_tdes_crypt_start()
547 		addr_out = dd->dma_addr_out;  in atmel_tdes_crypt_start()
549 		dd->flags &= ~TDES_FLAGS_FAST;  in atmel_tdes_crypt_start()
552 	dd->total -= count;  in atmel_tdes_crypt_start()
554 	if (dd->caps.has_dma)  in atmel_tdes_crypt_start()
555 		err = atmel_tdes_crypt_dma(dd, addr_in, addr_out, count);  in atmel_tdes_crypt_start()
557 		err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out, count);  in atmel_tdes_crypt_start()
559 	if (err && (dd->flags & TDES_FLAGS_FAST)) {  in atmel_tdes_crypt_start()
560 		dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_start()
561 		dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_start()
568 atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd)  in atmel_tdes_set_iv_as_last_ciphertext_block()  argument
570 	struct skcipher_request *req = dd->req;  in atmel_tdes_set_iv_as_last_ciphertext_block()
591 static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err)  in atmel_tdes_finish_req()  argument
593 	struct skcipher_request *req = dd->req;  in atmel_tdes_finish_req()
596 	clk_disable_unprepare(dd->iclk);  in atmel_tdes_finish_req()
598 	dd->flags &= ~TDES_FLAGS_BUSY;  in atmel_tdes_finish_req()
601 		atmel_tdes_set_iv_as_last_ciphertext_block(dd);  in atmel_tdes_finish_req()
606 static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd,  in atmel_tdes_handle_queue()  argument
615 	spin_lock_irqsave(&dd->lock, flags);  in atmel_tdes_handle_queue()
617 		ret = crypto_enqueue_request(&dd->queue, &req->base);  in atmel_tdes_handle_queue()
618 	if (dd->flags & TDES_FLAGS_BUSY) {  in atmel_tdes_handle_queue()
619 		spin_unlock_irqrestore(&dd->lock, flags);  in atmel_tdes_handle_queue()
622 	backlog = crypto_get_backlog(&dd->queue);  in atmel_tdes_handle_queue()
623 	async_req = crypto_dequeue_request(&dd->queue);  in atmel_tdes_handle_queue()
625 		dd->flags |= TDES_FLAGS_BUSY;  in atmel_tdes_handle_queue()
626 	spin_unlock_irqrestore(&dd->lock, flags);  in atmel_tdes_handle_queue()
637 	dd->req = req;  in atmel_tdes_handle_queue()
638 	dd->total = req->cryptlen;  in atmel_tdes_handle_queue()
639 	dd->in_offset = 0;  in atmel_tdes_handle_queue()
640 	dd->in_sg = req->src;  in atmel_tdes_handle_queue()
641 	dd->out_offset = 0;  in atmel_tdes_handle_queue()
642 	dd->out_sg = req->dst;  in atmel_tdes_handle_queue()
647 	dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode;  in atmel_tdes_handle_queue()
648 	dd->ctx = ctx;  in atmel_tdes_handle_queue()
649 	ctx->dd = dd;  in atmel_tdes_handle_queue()
651 	err = atmel_tdes_write_ctrl(dd);  in atmel_tdes_handle_queue()
653 		err = atmel_tdes_crypt_start(dd);  in atmel_tdes_handle_queue()
656 		atmel_tdes_finish_req(dd, err);  in atmel_tdes_handle_queue()
657 		tasklet_schedule(&dd->queue_task);  in atmel_tdes_handle_queue()
663 static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)  in atmel_tdes_crypt_dma_stop()  argument
668 	if (dd->flags & TDES_FLAGS_DMA) {  in atmel_tdes_crypt_dma_stop()
670 		if  (dd->flags & TDES_FLAGS_FAST) {  in atmel_tdes_crypt_dma_stop()
671 			dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);  in atmel_tdes_crypt_dma_stop()
672 			dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_dma_stop()
674 			dma_sync_single_for_device(dd->dev, dd->dma_addr_out,  in atmel_tdes_crypt_dma_stop()
675 				dd->dma_size, DMA_FROM_DEVICE);  in atmel_tdes_crypt_dma_stop()
678 			count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,  in atmel_tdes_crypt_dma_stop()
679 				dd->buf_out, dd->buflen, dd->dma_size, 1);  in atmel_tdes_crypt_dma_stop()
680 			if (count != dd->dma_size) {  in atmel_tdes_crypt_dma_stop()
741 	return atmel_tdes_handle_queue(ctx->dd, req);  in atmel_tdes_crypt()
744 static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd)  in atmel_tdes_dma_init()  argument
749 	dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx");  in atmel_tdes_dma_init()
750 	if (IS_ERR(dd->dma_lch_in.chan)) {  in atmel_tdes_dma_init()
751 		ret = PTR_ERR(dd->dma_lch_in.chan);  in atmel_tdes_dma_init()
755 	dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +  in atmel_tdes_dma_init()
757 	dd->dma_lch_in.dma_conf.src_maxburst = 1;  in atmel_tdes_dma_init()
758 	dd->dma_lch_in.dma_conf.src_addr_width =  in atmel_tdes_dma_init()
760 	dd->dma_lch_in.dma_conf.dst_maxburst = 1;  in atmel_tdes_dma_init()
761 	dd->dma_lch_in.dma_conf.dst_addr_width =  in atmel_tdes_dma_init()
763 	dd->dma_lch_in.dma_conf.device_fc = false;  in atmel_tdes_dma_init()
765 	dd->dma_lch_out.chan = dma_request_chan(dd->dev, "rx");  in atmel_tdes_dma_init()
766 	if (IS_ERR(dd->dma_lch_out.chan)) {  in atmel_tdes_dma_init()
767 		ret = PTR_ERR(dd->dma_lch_out.chan);  in atmel_tdes_dma_init()
771 	dd->dma_lch_out.dma_conf.src_addr = dd->phys_base +  in atmel_tdes_dma_init()
773 	dd->dma_lch_out.dma_conf.src_maxburst = 1;  in atmel_tdes_dma_init()
774 	dd->dma_lch_out.dma_conf.src_addr_width =  in atmel_tdes_dma_init()
776 	dd->dma_lch_out.dma_conf.dst_maxburst = 1;  in atmel_tdes_dma_init()
777 	dd->dma_lch_out.dma_conf.dst_addr_width =  in atmel_tdes_dma_init()
779 	dd->dma_lch_out.dma_conf.device_fc = false;  in atmel_tdes_dma_init()
784 	dma_release_channel(dd->dma_lch_in.chan);  in atmel_tdes_dma_init()
786 	dev_err(dd->dev, "no DMA channel available\n");  in atmel_tdes_dma_init()
790 static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd)  in atmel_tdes_dma_cleanup()  argument
792 	dma_release_channel(dd->dma_lch_in.chan);  in atmel_tdes_dma_cleanup()
793 	dma_release_channel(dd->dma_lch_out.chan);  in atmel_tdes_dma_cleanup()
900 	struct atmel_tdes_dev *dd;  in atmel_tdes_init_tfm()  local
904 	dd = atmel_tdes_find_dev(ctx);  in atmel_tdes_init_tfm()
905 	if (!dd)  in atmel_tdes_init_tfm()
1054 	struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data;  in atmel_tdes_queue_task()  local
1056 	atmel_tdes_handle_queue(dd, NULL);  in atmel_tdes_queue_task()
1061 	struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data;  in atmel_tdes_done_task()  local
1064 	if (!(dd->flags & TDES_FLAGS_DMA))  in atmel_tdes_done_task()
1065 		err = atmel_tdes_crypt_pdc_stop(dd);  in atmel_tdes_done_task()
1067 		err = atmel_tdes_crypt_dma_stop(dd);  in atmel_tdes_done_task()
1069 	if (dd->total && !err) {  in atmel_tdes_done_task()
1070 		if (dd->flags & TDES_FLAGS_FAST) {  in atmel_tdes_done_task()
1071 			dd->in_sg = sg_next(dd->in_sg);  in atmel_tdes_done_task()
1072 			dd->out_sg = sg_next(dd->out_sg);  in atmel_tdes_done_task()
1073 			if (!dd->in_sg || !dd->out_sg)  in atmel_tdes_done_task()
1077 			err = atmel_tdes_crypt_start(dd);  in atmel_tdes_done_task()
1082 	atmel_tdes_finish_req(dd, err);  in atmel_tdes_done_task()
1083 	atmel_tdes_handle_queue(dd, NULL);  in atmel_tdes_done_task()
1104 static void atmel_tdes_unregister_algs(struct atmel_tdes_dev *dd)  in atmel_tdes_unregister_algs()  argument
1112 static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd)  in atmel_tdes_register_algs()  argument
1133 static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd)  in atmel_tdes_get_cap()  argument
1136 	dd->caps.has_dma = 0;  in atmel_tdes_get_cap()
1137 	dd->caps.has_cfb_3keys = 0;  in atmel_tdes_get_cap()
1140 	switch (dd->hw_version & 0xf00) {  in atmel_tdes_get_cap()
1142 		dd->caps.has_dma = 1;  in atmel_tdes_get_cap()
1143 		dd->caps.has_cfb_3keys = 1;  in atmel_tdes_get_cap()
1148 		dev_warn(dd->dev,  in atmel_tdes_get_cap()