36 #define AE(handle, ae) handle->hal_handle->aes[ae]  argument
69 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,  in qat_hal_set_live_ctx()  argument
72 	AE(handle, ae).live_ctx_mask = ctx_mask;  in qat_hal_set_live_ctx()
76 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,  in qat_hal_rd_ae_csr()  argument
83 		value = GET_AE_CSR(handle, ae, csr);  in qat_hal_rd_ae_csr()
84 		if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))  in qat_hal_rd_ae_csr()
92 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,  in qat_hal_wr_ae_csr()  argument
99 		SET_AE_CSR(handle, ae, csr, value);  in qat_hal_wr_ae_csr()
100 		if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))  in qat_hal_wr_ae_csr()
108 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,  in qat_hal_get_wakeup_event()  argument
114 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);  in qat_hal_get_wakeup_event()
115 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);  in qat_hal_get_wakeup_event()
116 	*events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);  in qat_hal_get_wakeup_event()
117 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);  in qat_hal_get_wakeup_event()
120 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,  in qat_hal_wait_cycles()  argument
129 	base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);  in qat_hal_wait_cycles()
133 			csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);  in qat_hal_wait_cycles()
135 		cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);  in qat_hal_wait_cycles()
156 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,  in qat_hal_set_ae_ctx_mode()  argument
167 	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_set_ae_ctx_mode()
172 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);  in qat_hal_set_ae_ctx_mode()
176 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,  in qat_hal_set_ae_nn_mode()  argument
181 	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_set_ae_nn_mode()
189 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);  in qat_hal_set_ae_nn_mode()
194 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,  in qat_hal_set_ae_lm_mode()  argument
200 	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_set_ae_lm_mode()
219 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);  in qat_hal_set_ae_lm_mode()
272 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)  in qat_hal_reset()  argument
276 	ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);  in qat_hal_reset()
277 	ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;  in qat_hal_reset()
278 	ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;  in qat_hal_reset()
279 	SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);  in qat_hal_reset()
282 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,  in qat_hal_wr_indr_csr()  argument
288 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);  in qat_hal_wr_indr_csr()
293 		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);  in qat_hal_wr_indr_csr()
294 		qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);  in qat_hal_wr_indr_csr()
297 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);  in qat_hal_wr_indr_csr()
300 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,  in qat_hal_rd_indr_csr()  argument
306 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);  in qat_hal_rd_indr_csr()
307 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);  in qat_hal_rd_indr_csr()
308 	csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);  in qat_hal_rd_indr_csr()
309 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);  in qat_hal_rd_indr_csr()
314 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,  in qat_hal_put_sig_event()  argument
320 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);  in qat_hal_put_sig_event()
324 		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);  in qat_hal_put_sig_event()
325 		qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);  in qat_hal_put_sig_event()
327 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);  in qat_hal_put_sig_event()
330 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,  in qat_hal_put_wakeup_event()  argument
336 	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);  in qat_hal_put_wakeup_event()
340 		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);  in qat_hal_put_wakeup_event()
341 		qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,  in qat_hal_put_wakeup_event()
344 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);  in qat_hal_put_wakeup_event()
347 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)  in qat_hal_check_ae_alive()  argument
353 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {  in qat_hal_check_ae_alive()
354 		base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);  in qat_hal_check_ae_alive()
358 			cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);  in qat_hal_check_ae_alive()
371 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,  in qat_hal_check_ae_active()  argument
376 	enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_check_ae_active()
377 	active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);  in qat_hal_check_ae_active()
385 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)  in qat_hal_reset_timestamp()  argument
391 	misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);  in qat_hal_reset_timestamp()
393 		SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &  in qat_hal_reset_timestamp()
396 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {  in qat_hal_reset_timestamp()
397 		qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);  in qat_hal_reset_timestamp()
398 		qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);  in qat_hal_reset_timestamp()
401 	SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);  in qat_hal_reset_timestamp()
408 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)  in qat_hal_init_esram()  argument
411 			(void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +  in qat_hal_init_esram()
416 	if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)  in qat_hal_init_esram()
428 		qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);  in qat_hal_init_esram()
439 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)  in qat_hal_clr_reset()  argument
448 	ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);  in qat_hal_clr_reset()
449 	ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);  in qat_hal_clr_reset()
450 	ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);  in qat_hal_clr_reset()
452 		SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);  in qat_hal_clr_reset()
455 		csr = GET_GLB_CSR(handle, ICP_RESET);  in qat_hal_clr_reset()
456 	} while ((handle->hal_handle->ae_mask |  in qat_hal_clr_reset()
457 		 (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);  in qat_hal_clr_reset()
459 	clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);  in qat_hal_clr_reset()
460 	clk_csr |= handle->hal_handle->ae_mask << 0;  in qat_hal_clr_reset()
461 	clk_csr |= handle->hal_handle->slice_mask << 20;  in qat_hal_clr_reset()
462 	SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);  in qat_hal_clr_reset()
463 	if (qat_hal_check_ae_alive(handle))  in qat_hal_clr_reset()
467 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {  in qat_hal_clr_reset()
468 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,  in qat_hal_clr_reset()
470 		qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,  in qat_hal_clr_reset()
472 				    handle->hal_handle->upc_mask &  in qat_hal_clr_reset()
474 		qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);  in qat_hal_clr_reset()
475 		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);  in qat_hal_clr_reset()
476 		qat_hal_put_wakeup_event(handle, ae,  in qat_hal_clr_reset()
479 		qat_hal_put_sig_event(handle, ae,  in qat_hal_clr_reset()
483 	if (qat_hal_init_esram(handle))  in qat_hal_clr_reset()
485 	if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))  in qat_hal_clr_reset()
487 	qat_hal_reset_timestamp(handle);  in qat_hal_clr_reset()
495 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,  in qat_hal_disable_ctx()  argument
500 	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_disable_ctx()
503 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);  in qat_hal_disable_ctx()
536 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,  in qat_hal_wr_uwords()  argument
543 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);  in qat_hal_wr_uwords()
545 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);  in qat_hal_wr_uwords()
553 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);  in qat_hal_wr_uwords()
554 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);  in qat_hal_wr_uwords()
556 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);  in qat_hal_wr_uwords()
559 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,  in qat_hal_enable_ctx()  argument
564 	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_enable_ctx()
568 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);  in qat_hal_enable_ctx()
571 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)  in qat_hal_clear_xfer()  argument
576 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {  in qat_hal_clear_xfer()
578 			qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,  in qat_hal_clear_xfer()
580 			qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,  in qat_hal_clear_xfer()
586 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)  in qat_hal_clear_gpr()  argument
595 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {  in qat_hal_clear_gpr()
596 		csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);  in qat_hal_clear_gpr()
598 		qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);  in qat_hal_clear_gpr()
599 		csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_clear_gpr()
602 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);  in qat_hal_clear_gpr()
603 		qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),  in qat_hal_clear_gpr()
605 		qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,  in qat_hal_clear_gpr()
606 				    handle->hal_handle->upc_mask &  in qat_hal_clear_gpr()
608 		savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);  in qat_hal_clear_gpr()
609 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);  in qat_hal_clear_gpr()
610 		qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);  in qat_hal_clear_gpr()
611 		qat_hal_wr_indr_csr(handle, ae, ctx_mask,  in qat_hal_clear_gpr()
613 		qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);  in qat_hal_clear_gpr()
614 		qat_hal_enable_ctx(handle, ae, ctx_mask);  in qat_hal_clear_gpr()
616 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {  in qat_hal_clear_gpr()
619 			ret = qat_hal_wait_cycles(handle, ae, 20, 1);  in qat_hal_clear_gpr()
626 		qat_hal_disable_ctx(handle, ae, ctx_mask);  in qat_hal_clear_gpr()
627 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,  in qat_hal_clear_gpr()
629 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,  in qat_hal_clear_gpr()
631 		qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,  in qat_hal_clear_gpr()
632 				    handle->hal_handle->upc_mask &  in qat_hal_clear_gpr()
634 		qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);  in qat_hal_clear_gpr()
635 		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);  in qat_hal_clear_gpr()
636 		qat_hal_put_wakeup_event(handle, ae, ctx_mask,  in qat_hal_clear_gpr()
638 		qat_hal_put_sig_event(handle, ae, ctx_mask,  in qat_hal_clear_gpr()
652 	struct icp_qat_fw_loader_handle *handle;  in qat_hal_init()  local
659 	handle = kzalloc(sizeof(*handle), GFP_KERNEL);  in qat_hal_init()
660 	if (!handle)  in qat_hal_init()
663 	handle->hal_cap_g_ctl_csr_addr_v =  in qat_hal_init()
666 	handle->hal_cap_ae_xfer_csr_addr_v =  in qat_hal_init()
669 	handle->hal_ep_csr_addr_v =  in qat_hal_init()
672 	handle->hal_cap_ae_local_csr_addr_v =  in qat_hal_init()
673 		(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +  in qat_hal_init()
675 	handle->pci_dev = pci_info->pci_dev;  in qat_hal_init()
676 	if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) {  in qat_hal_init()
679 		handle->hal_sram_addr_v = sram_bar->virt_addr;  in qat_hal_init()
681 	handle->fw_auth = (handle->pci_dev->device ==  in qat_hal_init()
683 	handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);  in qat_hal_init()
684 	if (!handle->hal_handle)  in qat_hal_init()
686 	handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;  in qat_hal_init()
687 	handle->hal_handle->ae_mask = hw_data->ae_mask;  in qat_hal_init()
688 	handle->hal_handle->slice_mask = hw_data->accel_mask;  in qat_hal_init()
690 	handle->hal_handle->upc_mask = 0x1ffff;  in qat_hal_init()
691 	handle->hal_handle->max_ustore = 0x4000;  in qat_hal_init()
695 		handle->hal_handle->aes[ae].free_addr = 0;  in qat_hal_init()
696 		handle->hal_handle->aes[ae].free_size =  in qat_hal_init()
697 		    handle->hal_handle->max_ustore;  in qat_hal_init()
698 		handle->hal_handle->aes[ae].ustore_size =  in qat_hal_init()
699 		    handle->hal_handle->max_ustore;  in qat_hal_init()
700 		handle->hal_handle->aes[ae].live_ctx_mask =  in qat_hal_init()
704 	handle->hal_handle->ae_max_num = max_en_ae_id + 1;  in qat_hal_init()
706 	if (qat_hal_clr_reset(handle)) {  in qat_hal_init()
710 	qat_hal_clear_xfer(handle);  in qat_hal_init()
711 	if (!handle->fw_auth) {  in qat_hal_init()
712 		if (qat_hal_clear_gpr(handle))  in qat_hal_init()
717 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {  in qat_hal_init()
720 		csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);  in qat_hal_init()
722 		qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);  in qat_hal_init()
724 	accel_dev->fw_loader->fw_loader = handle;  in qat_hal_init()
728 	kfree(handle->hal_handle);  in qat_hal_init()
730 	kfree(handle);  in qat_hal_init()
734 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)  in qat_hal_deinit()  argument
736 	if (!handle)  in qat_hal_deinit()
738 	kfree(handle->hal_handle);  in qat_hal_deinit()
739 	kfree(handle);  in qat_hal_deinit()
742 void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,  in qat_hal_start()  argument
748 	if (handle->fw_auth) {  in qat_hal_start()
749 		SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START);  in qat_hal_start()
752 			fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);  in qat_hal_start()
759 		qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &  in qat_hal_start()
761 		qat_hal_enable_ctx(handle, ae, ctx_mask);  in qat_hal_start()
765 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,  in qat_hal_stop()  argument
768 	if (!handle->fw_auth)  in qat_hal_stop()
769 		qat_hal_disable_ctx(handle, ae, ctx_mask);  in qat_hal_stop()
772 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,  in qat_hal_set_pc()  argument
775 	qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,  in qat_hal_set_pc()
776 			    handle->hal_handle->upc_mask & upc);  in qat_hal_set_pc()
779 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,  in qat_hal_get_uwords()  argument
786 	misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);  in qat_hal_get_uwords()
787 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,  in qat_hal_get_uwords()
789 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);  in qat_hal_get_uwords()
792 		qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);  in qat_hal_get_uwords()
794 		uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);  in qat_hal_get_uwords()
795 		uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);  in qat_hal_get_uwords()
799 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);  in qat_hal_get_uwords()
800 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);  in qat_hal_get_uwords()
803 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,  in qat_hal_wr_umem()  argument
809 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);  in qat_hal_wr_umem()
811 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);  in qat_hal_wr_umem()
822 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);  in qat_hal_wr_umem()
823 		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);  in qat_hal_wr_umem()
825 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);  in qat_hal_wr_umem()
829 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,  in qat_hal_exec_micro_inst()  argument
845 	if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {  in qat_hal_exec_micro_inst()
850 	ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);  in qat_hal_exec_micro_inst()
851 	ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);  in qat_hal_exec_micro_inst()
852 	ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,  in qat_hal_exec_micro_inst()
854 	ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,  in qat_hal_exec_micro_inst()
857 		qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);  in qat_hal_exec_micro_inst()
858 	qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);  in qat_hal_exec_micro_inst()
859 	savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);  in qat_hal_exec_micro_inst()
860 	savpc = (savpc & handle->hal_handle->upc_mask) >> 0;  in qat_hal_exec_micro_inst()
861 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_exec_micro_inst()
863 	savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);  in qat_hal_exec_micro_inst()
864 	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);  in qat_hal_exec_micro_inst()
865 	ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);  in qat_hal_exec_micro_inst()
866 	ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,  in qat_hal_exec_micro_inst()
868 	ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,  in qat_hal_exec_micro_inst()
870 	act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);  in qat_hal_exec_micro_inst()
872 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);  in qat_hal_exec_micro_inst()
873 	qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);  in qat_hal_exec_micro_inst()
874 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);  in qat_hal_exec_micro_inst()
875 	qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);  in qat_hal_exec_micro_inst()
877 		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);  in qat_hal_exec_micro_inst()
878 	qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);  in qat_hal_exec_micro_inst()
879 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);  in qat_hal_exec_micro_inst()
880 	qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);  in qat_hal_exec_micro_inst()
881 	qat_hal_enable_ctx(handle, ae, (1 << ctx));  in qat_hal_exec_micro_inst()
883 	if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)  in qat_hal_exec_micro_inst()
888 		ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,  in qat_hal_exec_micro_inst()
890 		*endpc = ctx_status & handle->hal_handle->upc_mask;  in qat_hal_exec_micro_inst()
893 	qat_hal_disable_ctx(handle, ae, (1 << ctx));  in qat_hal_exec_micro_inst()
895 		qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);  in qat_hal_exec_micro_inst()
896 	qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);  in qat_hal_exec_micro_inst()
897 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,  in qat_hal_exec_micro_inst()
898 			    handle->hal_handle->upc_mask & savpc);  in qat_hal_exec_micro_inst()
899 	csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);  in qat_hal_exec_micro_inst()
901 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);  in qat_hal_exec_micro_inst()
902 	qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);  in qat_hal_exec_micro_inst()
903 	qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);  in qat_hal_exec_micro_inst()
904 	qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);  in qat_hal_exec_micro_inst()
905 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),  in qat_hal_exec_micro_inst()
907 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),  in qat_hal_exec_micro_inst()
909 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),  in qat_hal_exec_micro_inst()
911 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),  in qat_hal_exec_micro_inst()
913 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),  in qat_hal_exec_micro_inst()
915 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),  in qat_hal_exec_micro_inst()
917 	qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);  in qat_hal_exec_micro_inst()
918 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);  in qat_hal_exec_micro_inst()
923 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,  in qat_hal_rd_rel_reg()  argument
947 	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);  in qat_hal_rd_rel_reg()
948 	ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);  in qat_hal_rd_rel_reg()
949 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_rd_rel_reg()
952 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,  in qat_hal_rd_rel_reg()
954 	qat_hal_get_uwords(handle, ae, 0, 1, &savuword);  in qat_hal_rd_rel_reg()
955 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);  in qat_hal_rd_rel_reg()
956 	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);  in qat_hal_rd_rel_reg()
958 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);  in qat_hal_rd_rel_reg()
962 	qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);  in qat_hal_rd_rel_reg()
963 	qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);  in qat_hal_rd_rel_reg()
964 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);  in qat_hal_rd_rel_reg()
966 	qat_hal_wait_cycles(handle, ae, 0x8, 0);  in qat_hal_rd_rel_reg()
972 	*data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);  in qat_hal_rd_rel_reg()
973 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);  in qat_hal_rd_rel_reg()
974 	qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);  in qat_hal_rd_rel_reg()
976 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,  in qat_hal_rd_rel_reg()
978 	qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);  in qat_hal_rd_rel_reg()
979 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);  in qat_hal_rd_rel_reg()
984 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,  in qat_hal_wr_rel_reg()  argument
1027 	return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,  in qat_hal_wr_rel_reg()
1065 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,  in qat_hal_exec_micro_init_lm()  argument
1075 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);  in qat_hal_exec_micro_init_lm()
1076 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);  in qat_hal_exec_micro_init_lm()
1077 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);  in qat_hal_exec_micro_init_lm()
1078 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);  in qat_hal_exec_micro_init_lm()
1079 		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);  in qat_hal_exec_micro_init_lm()
1082 	stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,  in qat_hal_exec_micro_init_lm()
1086 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);  in qat_hal_exec_micro_init_lm()
1087 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);  in qat_hal_exec_micro_init_lm()
1088 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);  in qat_hal_exec_micro_init_lm()
1089 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);  in qat_hal_exec_micro_init_lm()
1090 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);  in qat_hal_exec_micro_init_lm()
1095 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,  in qat_hal_batch_wr_lm()  argument
1108 	if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)  in qat_hal_batch_wr_lm()
1109 		alloc_inst_size = handle->hal_handle->max_ustore;  in qat_hal_batch_wr_lm()
1130 		stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,  in qat_hal_batch_wr_lm()
1138 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,  in qat_hal_put_rel_rd_xfer()  argument
1149 	status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_put_rel_rd_xfer()
1166 		SET_AE_XFER(handle, ae, reg_addr, val);  in qat_hal_put_rel_rd_xfer()
1170 		SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);  in qat_hal_put_rel_rd_xfer()
1179 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,  in qat_hal_put_rel_wr_xfer()  argument
1199 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_put_rel_wr_xfer()
1216 	qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);  in qat_hal_put_rel_wr_xfer()
1230 	status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,  in qat_hal_put_rel_wr_xfer()
1232 	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);  in qat_hal_put_rel_wr_xfer()
1236 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,  in qat_hal_put_rel_nn()  argument
1243 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_put_rel_nn()
1245 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);  in qat_hal_put_rel_nn()
1247 	stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);  in qat_hal_put_rel_nn()
1248 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);  in qat_hal_put_rel_nn()
1253 				      *handle, unsigned char ae,  in qat_hal_convert_abs_to_rel()
1260 	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);  in qat_hal_convert_abs_to_rel()
1273 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,  in qat_hal_init_gpr()  argument
1288 			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,  in qat_hal_init_gpr()
1297 		stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);  in qat_hal_init_gpr()
1307 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,  in qat_hal_init_wr_xfer()  argument
1322 			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,  in qat_hal_init_wr_xfer()
1331 		stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,  in qat_hal_init_wr_xfer()
1342 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,  in qat_hal_init_rd_xfer()  argument
1357 			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,  in qat_hal_init_rd_xfer()
1366 		stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,  in qat_hal_init_rd_xfer()
1377 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,  in qat_hal_init_nn()  argument
1390 		stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);  in qat_hal_init_nn()