40 static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring)  in adf_reserve_ring()  argument
43 	if (bank->ring_mask & (1 << ring)) {  in adf_reserve_ring()
47 	bank->ring_mask |= (1 << ring);  in adf_reserve_ring()
52 static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring)  in adf_unreserve_ring()  argument
55 	bank->ring_mask &= ~(1 << ring);  in adf_unreserve_ring()
59 static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)  in adf_enable_ring_irq()  argument
64 	bank->irq_mask |= (1 << ring);  in adf_enable_ring_irq()
72 static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)  in adf_disable_ring_irq()  argument
77 	bank->irq_mask &= ~(1 << ring);  in adf_disable_ring_irq()
83 bool adf_ring_nearly_full(struct adf_etr_ring_data *ring)  in adf_ring_nearly_full()  argument
85 	return atomic_read(ring->inflights) > ring->threshold;  in adf_ring_nearly_full()
88 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)  in adf_send_message()  argument
90 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);  in adf_send_message()
92 	if (atomic_add_return(1, ring->inflights) >  in adf_send_message()
93 	    ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {  in adf_send_message()
94 		atomic_dec(ring->inflights);  in adf_send_message()
97 	spin_lock_bh(&ring->lock);  in adf_send_message()
98 	memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg,  in adf_send_message()
99 	       ADF_MSG_SIZE_TO_BYTES(ring->msg_size));  in adf_send_message()
101 	ring->tail = adf_modulo(ring->tail +  in adf_send_message()
102 				ADF_MSG_SIZE_TO_BYTES(ring->msg_size),  in adf_send_message()
103 				ADF_RING_SIZE_MODULO(ring->ring_size));  in adf_send_message()
104 	csr_ops->write_csr_ring_tail(ring->bank->csr_addr,  in adf_send_message()
105 				     ring->bank->bank_number, ring->ring_number,  in adf_send_message()
106 				     ring->tail);  in adf_send_message()
107 	spin_unlock_bh(&ring->lock);  in adf_send_message()
112 static int adf_handle_response(struct adf_etr_ring_data *ring)  in adf_handle_response()  argument
114 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);  in adf_handle_response()
116 	u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);  in adf_handle_response()
119 		ring->callback((u32 *)msg);  in adf_handle_response()
120 		atomic_dec(ring->inflights);  in adf_handle_response()
122 		ring->head = adf_modulo(ring->head +  in adf_handle_response()
123 					ADF_MSG_SIZE_TO_BYTES(ring->msg_size),  in adf_handle_response()
124 					ADF_RING_SIZE_MODULO(ring->ring_size));  in adf_handle_response()
126 		msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);  in adf_handle_response()
129 		csr_ops->write_csr_ring_head(ring->bank->csr_addr,  in adf_handle_response()
130 					     ring->bank->bank_number,  in adf_handle_response()
131 					     ring->ring_number, ring->head);  in adf_handle_response()
136 static void adf_configure_tx_ring(struct adf_etr_ring_data *ring)  in adf_configure_tx_ring()  argument
138 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);  in adf_configure_tx_ring()
139 	u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);  in adf_configure_tx_ring()
141 	csr_ops->write_csr_ring_config(ring->bank->csr_addr,  in adf_configure_tx_ring()
142 				       ring->bank->bank_number,  in adf_configure_tx_ring()
143 				       ring->ring_number, ring_config);  in adf_configure_tx_ring()
147 static void adf_configure_rx_ring(struct adf_etr_ring_data *ring)  in adf_configure_rx_ring()  argument
149 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);  in adf_configure_rx_ring()
151 			BUILD_RESP_RING_CONFIG(ring->ring_size,  in adf_configure_rx_ring()
155 	csr_ops->write_csr_ring_config(ring->bank->csr_addr,  in adf_configure_rx_ring()
156 				       ring->bank->bank_number,  in adf_configure_rx_ring()
157 				       ring->ring_number, ring_config);  in adf_configure_rx_ring()
160 static int adf_init_ring(struct adf_etr_ring_data *ring)  in adf_init_ring()  argument
162 	struct adf_etr_bank_data *bank = ring->bank;  in adf_init_ring()
168 			ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);  in adf_init_ring()
171 	ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev),  in adf_init_ring()
172 					     ring_size_bytes, &ring->dma_addr,  in adf_init_ring()
174 	if (!ring->base_addr)  in adf_init_ring()
177 	memset(ring->base_addr, 0x7F, ring_size_bytes);  in adf_init_ring()
179 	if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) {  in adf_init_ring()
180 		dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n");  in adf_init_ring()
182 				  ring->base_addr, ring->dma_addr);  in adf_init_ring()
183 		ring->base_addr = NULL;  in adf_init_ring()
187 	if (hw_data->tx_rings_mask & (1 << ring->ring_number))  in adf_init_ring()
188 		adf_configure_tx_ring(ring);  in adf_init_ring()
191 		adf_configure_rx_ring(ring);  in adf_init_ring()
193 	ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr,  in adf_init_ring()
194 						      ring->ring_size);  in adf_init_ring()
196 	csr_ops->write_csr_ring_base(ring->bank->csr_addr,  in adf_init_ring()
197 				     ring->bank->bank_number, ring->ring_number,  in adf_init_ring()
199 	spin_lock_init(&ring->lock);  in adf_init_ring()
203 static void adf_cleanup_ring(struct adf_etr_ring_data *ring)  in adf_cleanup_ring()  argument
206 			ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);  in adf_cleanup_ring()
209 	if (ring->base_addr) {  in adf_cleanup_ring()
210 		memset(ring->base_addr, 0x7F, ring_size_bytes);  in adf_cleanup_ring()
211 		dma_free_coherent(&GET_DEV(ring->bank->accel_dev),  in adf_cleanup_ring()
212 				  ring_size_bytes, ring->base_addr,  in adf_cleanup_ring()
213 				  ring->dma_addr);  in adf_cleanup_ring()
226 	struct adf_etr_ring_data *ring;  in adf_create_ring()  local
243 			"Invalid ring size for given msg size\n");  in adf_create_ring()
252 		dev_err(&GET_DEV(accel_dev), "Can't get ring number\n");  in adf_create_ring()
256 		dev_err(&GET_DEV(accel_dev), "Invalid ring number\n");  in adf_create_ring()
263 		dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n",  in adf_create_ring()
267 	ring = &bank->rings[ring_num];  in adf_create_ring()
268 	ring->ring_number = ring_num;  in adf_create_ring()
269 	ring->bank = bank;  in adf_create_ring()
270 	ring->callback = callback;  in adf_create_ring()
271 	ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size);  in adf_create_ring()
272 	ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);  in adf_create_ring()
273 	ring->head = 0;  in adf_create_ring()
274 	ring->tail = 0;  in adf_create_ring()
275 	max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);  in adf_create_ring()
276 	ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD);  in adf_create_ring()
277 	atomic_set(ring->inflights, 0);  in adf_create_ring()
278 	ret = adf_init_ring(ring);  in adf_create_ring()
282 	/* Enable HW arbitration for the given ring */  in adf_create_ring()
283 	adf_update_ring_arb(ring);  in adf_create_ring()
285 	if (adf_ring_debugfs_add(ring, ring_name)) {  in adf_create_ring()
287 			"Couldn't add ring debugfs entry\n");  in adf_create_ring()
294 		adf_enable_ring_irq(bank, ring->ring_number);  in adf_create_ring()
295 	*ring_ptr = ring;  in adf_create_ring()
298 	adf_cleanup_ring(ring);  in adf_create_ring()
300 	adf_update_ring_arb(ring);  in adf_create_ring()
304 void adf_remove_ring(struct adf_etr_ring_data *ring)  in adf_remove_ring()  argument
306 	struct adf_etr_bank_data *bank = ring->bank;  in adf_remove_ring()
309 	/* Disable interrupts for the given ring */  in adf_remove_ring()
310 	adf_disable_ring_irq(bank, ring->ring_number);  in adf_remove_ring()
315 				       ring->ring_number, 0);  in adf_remove_ring()
317 				     ring->ring_number, 0);  in adf_remove_ring()
318 	adf_ring_debugfs_rm(ring);  in adf_remove_ring()
319 	adf_unreserve_ring(bank, ring->ring_number);  in adf_remove_ring()
320 	/* Disable HW arbitration for the given ring */  in adf_remove_ring()
321 	adf_update_ring_arb(ring);  in adf_remove_ring()
322 	adf_cleanup_ring(ring);  in adf_remove_ring()
392 	struct adf_etr_ring_data *ring;  in adf_init_bank()  local
425 		ring = &bank->rings[i];  in adf_init_bank()
427 			ring->inflights =  in adf_init_bank()
431 			if (!ring->inflights)  in adf_init_bank()
440 			ring->inflights = tx_ring->inflights;  in adf_init_bank()
456 		ring = &bank->rings[i];  in adf_init_bank()
457 		kfree(ring->inflights);  in adf_init_bank()
458 		ring->inflights = NULL;  in adf_init_bank()
532 		struct adf_etr_ring_data *ring = &bank->rings[i];  in cleanup_bank()  local
535 			adf_cleanup_ring(ring);  in cleanup_bank()
538 			kfree(ring->inflights);  in cleanup_bank()