36 static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring)  in adf_reserve_ring()  argument
39 	if (bank->ring_mask & (1 << ring)) {  in adf_reserve_ring()
43 	bank->ring_mask |= (1 << ring);  in adf_reserve_ring()
48 static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring)  in adf_unreserve_ring()  argument
51 	bank->ring_mask &= ~(1 << ring);  in adf_unreserve_ring()
55 static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)  in adf_enable_ring_irq()  argument
58 	bank->irq_mask |= (1 << ring);  in adf_enable_ring_irq()
65 static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)  in adf_disable_ring_irq()  argument
68 	bank->irq_mask &= ~(1 << ring);  in adf_disable_ring_irq()
73 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)  in adf_send_message()  argument
75 	if (atomic_add_return(1, ring->inflights) >  in adf_send_message()
76 	    ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {  in adf_send_message()
77 		atomic_dec(ring->inflights);  in adf_send_message()
80 	spin_lock_bh(&ring->lock);  in adf_send_message()
81 	memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg,  in adf_send_message()
82 	       ADF_MSG_SIZE_TO_BYTES(ring->msg_size));  in adf_send_message()
84 	ring->tail = adf_modulo(ring->tail +  in adf_send_message()
85 				ADF_MSG_SIZE_TO_BYTES(ring->msg_size),  in adf_send_message()
86 				ADF_RING_SIZE_MODULO(ring->ring_size));  in adf_send_message()
87 	WRITE_CSR_RING_TAIL(ring->bank->csr_addr, ring->bank->bank_number,  in adf_send_message()
88 			    ring->ring_number, ring->tail);  in adf_send_message()
89 	spin_unlock_bh(&ring->lock);  in adf_send_message()
93 static int adf_handle_response(struct adf_etr_ring_data *ring)  in adf_handle_response()  argument
96 	u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);  in adf_handle_response()
99 		ring->callback((u32 *)msg);  in adf_handle_response()
100 		atomic_dec(ring->inflights);  in adf_handle_response()
102 		ring->head = adf_modulo(ring->head +  in adf_handle_response()
103 					ADF_MSG_SIZE_TO_BYTES(ring->msg_size),  in adf_handle_response()
104 					ADF_RING_SIZE_MODULO(ring->ring_size));  in adf_handle_response()
106 		msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);  in adf_handle_response()
109 		WRITE_CSR_RING_HEAD(ring->bank->csr_addr,  in adf_handle_response()
110 				    ring->bank->bank_number,  in adf_handle_response()
111 				    ring->ring_number, ring->head);  in adf_handle_response()
115 static void adf_configure_tx_ring(struct adf_etr_ring_data *ring)  in adf_configure_tx_ring()  argument
117 	u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);  in adf_configure_tx_ring()
119 	WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number,  in adf_configure_tx_ring()
120 			      ring->ring_number, ring_config);  in adf_configure_tx_ring()
123 static void adf_configure_rx_ring(struct adf_etr_ring_data *ring)  in adf_configure_rx_ring()  argument
126 			BUILD_RESP_RING_CONFIG(ring->ring_size,  in adf_configure_rx_ring()
130 	WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number,  in adf_configure_rx_ring()
131 			      ring->ring_number, ring_config);  in adf_configure_rx_ring()
134 static int adf_init_ring(struct adf_etr_ring_data *ring)  in adf_init_ring()  argument
136 	struct adf_etr_bank_data *bank = ring->bank;  in adf_init_ring()
141 			ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);  in adf_init_ring()
144 	ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev),  in adf_init_ring()
145 					     ring_size_bytes, &ring->dma_addr,  in adf_init_ring()
147 	if (!ring->base_addr)  in adf_init_ring()
150 	memset(ring->base_addr, 0x7F, ring_size_bytes);  in adf_init_ring()
152 	if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) {  in adf_init_ring()
153 		dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n");  in adf_init_ring()
155 				  ring->base_addr, ring->dma_addr);  in adf_init_ring()
159 	if (hw_data->tx_rings_mask & (1 << ring->ring_number))  in adf_init_ring()
160 		adf_configure_tx_ring(ring);  in adf_init_ring()
163 		adf_configure_rx_ring(ring);  in adf_init_ring()
165 	ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size);  in adf_init_ring()
166 	WRITE_CSR_RING_BASE(ring->bank->csr_addr, ring->bank->bank_number,  in adf_init_ring()
167 			    ring->ring_number, ring_base);  in adf_init_ring()
168 	spin_lock_init(&ring->lock);  in adf_init_ring()
172 static void adf_cleanup_ring(struct adf_etr_ring_data *ring)  in adf_cleanup_ring()  argument
175 			ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);  in adf_cleanup_ring()
178 	if (ring->base_addr) {  in adf_cleanup_ring()
179 		memset(ring->base_addr, 0x7F, ring_size_bytes);  in adf_cleanup_ring()
180 		dma_free_coherent(&GET_DEV(ring->bank->accel_dev),  in adf_cleanup_ring()
181 				  ring_size_bytes, ring->base_addr,  in adf_cleanup_ring()
182 				  ring->dma_addr);  in adf_cleanup_ring()
194 	struct adf_etr_ring_data *ring;  in adf_create_ring()  local
210 			"Invalid ring size for given msg size\n");  in adf_create_ring()
219 		dev_err(&GET_DEV(accel_dev), "Can't get ring number\n");  in adf_create_ring()
223 		dev_err(&GET_DEV(accel_dev), "Invalid ring number\n");  in adf_create_ring()
229 		dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n",  in adf_create_ring()
233 	ring = &bank->rings[ring_num];  in adf_create_ring()
234 	ring->ring_number = ring_num;  in adf_create_ring()
235 	ring->bank = bank;  in adf_create_ring()
236 	ring->callback = callback;  in adf_create_ring()
237 	ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size);  in adf_create_ring()
238 	ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);  in adf_create_ring()
239 	ring->head = 0;  in adf_create_ring()
240 	ring->tail = 0;  in adf_create_ring()
241 	atomic_set(ring->inflights, 0);  in adf_create_ring()
242 	ret = adf_init_ring(ring);  in adf_create_ring()
246 	/* Enable HW arbitration for the given ring */  in adf_create_ring()
247 	adf_update_ring_arb(ring);  in adf_create_ring()
249 	if (adf_ring_debugfs_add(ring, ring_name)) {  in adf_create_ring()
251 			"Couldn't add ring debugfs entry\n");  in adf_create_ring()
258 		adf_enable_ring_irq(bank, ring->ring_number);  in adf_create_ring()
259 	*ring_ptr = ring;  in adf_create_ring()
262 	adf_cleanup_ring(ring);  in adf_create_ring()
264 	adf_update_ring_arb(ring);  in adf_create_ring()
268 void adf_remove_ring(struct adf_etr_ring_data *ring)  in adf_remove_ring()  argument
270 	struct adf_etr_bank_data *bank = ring->bank;  in adf_remove_ring()
272 	/* Disable interrupts for the given ring */  in adf_remove_ring()
273 	adf_disable_ring_irq(bank, ring->ring_number);  in adf_remove_ring()
277 			      ring->ring_number, 0);  in adf_remove_ring()
279 			    ring->ring_number, 0);  in adf_remove_ring()
280 	adf_ring_debugfs_rm(ring);  in adf_remove_ring()
281 	adf_unreserve_ring(bank, ring->ring_number);  in adf_remove_ring()
282 	/* Disable HW arbitration for the given ring */  in adf_remove_ring()
283 	adf_update_ring_arb(ring);  in adf_remove_ring()
284 	adf_cleanup_ring(ring);  in adf_remove_ring()
346 	struct adf_etr_ring_data *ring;  in adf_init_bank()  local
369 		ring = &bank->rings[i];  in adf_init_bank()
371 			ring->inflights =  in adf_init_bank()
375 			if (!ring->inflights)  in adf_init_bank()
384 			ring->inflights = tx_ring->inflights;  in adf_init_bank()
398 		ring = &bank->rings[i];  in adf_init_bank()
400 			kfree(ring->inflights);  in adf_init_bank()
472 		struct adf_etr_ring_data *ring = &bank->rings[i];  in cleanup_bank()  local
475 			adf_cleanup_ring(ring);  in cleanup_bank()
478 			kfree(ring->inflights);  in cleanup_bank()