1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4  * Copyright (C) 2019-2022 Linaro Ltd.
12 #include <linux/dma-direction.h>
44  * transaction.  If a buffer in an operation must be mapped for DMA, this is
75 	__le64 addr;		/* DMA address */
81 /* gsi_tre->flags mask values (in CPU byte order) */
87 int gsi_trans_pool_init(struct gsi_trans_pool *pool, size_t size, u32 count,  in gsi_trans_pool_init()  argument
93 		return -EINVAL;  in gsi_trans_pool_init()
95 		return -EINVAL;  in gsi_trans_pool_init()
97 		return -EINVAL;  in gsi_trans_pool_init()
99 	/* By allocating a few extra entries in our pool (one less  in gsi_trans_pool_init()
102 	 * ever worrying about straddling the end of the pool array.  in gsi_trans_pool_init()
104 	 * we just allocate free entries from the beginning of the pool.  in gsi_trans_pool_init()
106 	virt = kcalloc(count + max_alloc - 1, size, GFP_KERNEL);  in gsi_trans_pool_init()
108 		return -ENOMEM;  in gsi_trans_pool_init()
110 	pool->base = virt;  in gsi_trans_pool_init()
112 	pool->count = ksize(pool->base) / size;  in gsi_trans_pool_init()
113 	pool->free = 0;  in gsi_trans_pool_init()
114 	pool->max_alloc = max_alloc;  in gsi_trans_pool_init()
115 	pool->size = size;  in gsi_trans_pool_init()
116 	pool->addr = 0;		/* Only used for DMA pools */  in gsi_trans_pool_init()
121 void gsi_trans_pool_exit(struct gsi_trans_pool *pool)  in gsi_trans_pool_exit()  argument
123 	kfree(pool->base);  in gsi_trans_pool_exit()
124 	memset(pool, 0, sizeof(*pool));  in gsi_trans_pool_exit()
127 /* Home-grown DMA pool.  This way we can preallocate the pool, and guarantee
129  * require up to max_alloc elements from the pool.  But we only allow
130  * allocation of a single element from a DMA pool at a time.
132 int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool,  in gsi_trans_pool_init_dma()  argument
140 		return -EINVAL;  in gsi_trans_pool_init_dma()
142 		return -EINVAL;  in gsi_trans_pool_init_dma()
144 		return -EINVAL;  in gsi_trans_pool_init_dma()
146 	/* Don't let allocations cross a power-of-two boundary */  in gsi_trans_pool_init_dma()
148 	total_size = (count + max_alloc - 1) * size;  in gsi_trans_pool_init_dma()
150 	/* The allocator will give us a power-of-2 number of pages  in gsi_trans_pool_init_dma()
160 		return -ENOMEM;  in gsi_trans_pool_init_dma()
162 	pool->base = virt;  in gsi_trans_pool_init_dma()
163 	pool->count = total_size / size;  in gsi_trans_pool_init_dma()
164 	pool->free = 0;  in gsi_trans_pool_init_dma()
165 	pool->size = size;  in gsi_trans_pool_init_dma()
166 	pool->max_alloc = max_alloc;  in gsi_trans_pool_init_dma()
167 	pool->addr = addr;  in gsi_trans_pool_init_dma()
172 void gsi_trans_pool_exit_dma(struct device *dev, struct gsi_trans_pool *pool)  in gsi_trans_pool_exit_dma()  argument
174 	size_t total_size = pool->count * pool->size;  in gsi_trans_pool_exit_dma()
176 	dma_free_coherent(dev, total_size, pool->base, pool->addr);  in gsi_trans_pool_exit_dma()
177 	memset(pool, 0, sizeof(*pool));  in gsi_trans_pool_exit_dma()
180 /* Return the byte offset of the next free entry in the pool */
181 static u32 gsi_trans_pool_alloc_common(struct gsi_trans_pool *pool, u32 count)  in gsi_trans_pool_alloc_common()  argument
186 	WARN_ON(count > pool->max_alloc);  in gsi_trans_pool_alloc_common()
189 	if (count > pool->count - pool->free)  in gsi_trans_pool_alloc_common()
190 		pool->free = 0;  in gsi_trans_pool_alloc_common()
192 	offset = pool->free * pool->size;  in gsi_trans_pool_alloc_common()
193 	pool->free += count;  in gsi_trans_pool_alloc_common()
194 	memset(pool->base + offset, 0, count * pool->size);  in gsi_trans_pool_alloc_common()
199 /* Allocate a contiguous block of zeroed entries from a pool */
200 void *gsi_trans_pool_alloc(struct gsi_trans_pool *pool, u32 count)  in gsi_trans_pool_alloc()  argument
202 	return pool->base + gsi_trans_pool_alloc_common(pool, count);  in gsi_trans_pool_alloc()
205 /* Allocate a single zeroed entry from a DMA pool */
206 void *gsi_trans_pool_alloc_dma(struct gsi_trans_pool *pool, dma_addr_t *addr)  in gsi_trans_pool_alloc_dma()  argument
208 	u32 offset = gsi_trans_pool_alloc_common(pool, 1);  in gsi_trans_pool_alloc_dma()
210 	*addr = pool->addr + offset;  in gsi_trans_pool_alloc_dma()
212 	return pool->base + offset;  in gsi_trans_pool_alloc_dma()
218 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_map()
221 	index += trans->used_count - 1;  in gsi_trans_map()
224 	channel->trans_info.map[index % channel->tre_ring.count] = trans;  in gsi_trans_map()
232 	return channel->trans_info.map[index % channel->tre_ring.count];  in gsi_channel_trans_mapped()
238 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_channel_trans_complete()
239 	u16 trans_id = trans_info->completed_id;  in gsi_channel_trans_complete()
241 	if (trans_id == trans_info->pending_id) {  in gsi_channel_trans_complete()
243 		if (trans_id == trans_info->pending_id)  in gsi_channel_trans_complete()
247 	return &trans_info->trans[trans_id %= channel->tre_count];  in gsi_channel_trans_complete()
253 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_committed()
254 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_trans_move_committed()
257 	trans_info->allocated_id++;  in gsi_trans_move_committed()
263 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_pending()
264 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_trans_move_pending()
265 	u16 trans_index = trans - &trans_info->trans[0];  in gsi_trans_move_pending()
269 	delta = trans_index - trans_info->committed_id + 1;  in gsi_trans_move_pending()
270 	trans_info->committed_id += delta % channel->tre_count;  in gsi_trans_move_pending()
276 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_complete()
277 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_trans_move_complete()
278 	u16 trans_index = trans - trans_info->trans;  in gsi_trans_move_complete()
282 	delta = trans_index - trans_info->pending_id + 1;  in gsi_trans_move_complete()
283 	delta %= channel->tre_count;  in gsi_trans_move_complete()
284 	trans_info->pending_id += delta;  in gsi_trans_move_complete()
290 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_polled()
291 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_trans_move_polled()
294 	trans_info->completed_id++;  in gsi_trans_move_polled()
301 	int avail = atomic_read(&trans_info->tre_avail);  in gsi_trans_tre_reserve()
305 		new = avail - (int)tre_count;  in gsi_trans_tre_reserve()
308 	} while (!atomic_try_cmpxchg(&trans_info->tre_avail, &avail, new));  in gsi_trans_tre_reserve()
313 /* Release previously-reserved TRE entries to a channel */
317 	atomic_add(tre_count, &trans_info->tre_avail);  in gsi_trans_tre_release()
326 	trans_info = &gsi->channel[channel_id].trans_info;  in gsi_channel_trans_idle()
328 	return atomic_read(&trans_info->tre_avail) == tre_max;  in gsi_channel_trans_idle()
336 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_trans_alloc()
341 	if (WARN_ON(tre_count > channel->trans_tre_max))  in gsi_channel_trans_alloc()
344 	trans_info = &channel->trans_info;  in gsi_channel_trans_alloc()
350 	trans_index = trans_info->free_id % channel->tre_count;  in gsi_channel_trans_alloc()
351 	trans = &trans_info->trans[trans_index];  in gsi_channel_trans_alloc()
354 	/* Initialize non-zero fields in the transaction */  in gsi_channel_trans_alloc()
355 	trans->gsi = gsi;  in gsi_channel_trans_alloc()
356 	trans->channel_id = channel_id;  in gsi_channel_trans_alloc()
357 	trans->rsvd_count = tre_count;  in gsi_channel_trans_alloc()
358 	init_completion(&trans->completion);  in gsi_channel_trans_alloc()
361 	trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count);  in gsi_channel_trans_alloc()
362 	sg_init_marker(trans->sgl, tre_count);  in gsi_channel_trans_alloc()
364 	trans->direction = direction;  in gsi_channel_trans_alloc()
365 	refcount_set(&trans->refcount, 1);  in gsi_channel_trans_alloc()
368 	trans_info->free_id++;  in gsi_channel_trans_alloc()
373 /* Free a previously-allocated transaction */
378 	if (!refcount_dec_and_test(&trans->refcount))  in gsi_trans_free()
384 	trans_info = &trans->gsi->channel[trans->channel_id].trans_info;  in gsi_trans_free()
385 	if (!trans->used_count) {  in gsi_trans_free()
386 		trans_info->allocated_id++;  in gsi_trans_free()
387 		trans_info->committed_id++;  in gsi_trans_free()
388 		trans_info->pending_id++;  in gsi_trans_free()
389 		trans_info->completed_id++;  in gsi_trans_free()
395 	trans_info->polled_id++;  in gsi_trans_free()
400 	gsi_trans_tre_release(trans_info, trans->rsvd_count);  in gsi_trans_free()
407 	u32 which = trans->used_count++;  in gsi_trans_cmd_add()
410 	WARN_ON(which >= trans->rsvd_count);  in gsi_trans_cmd_add()
413 	 * Their payloads come from a pool whose memory is allocated  in gsi_trans_cmd_add()
415 	 * for DMA (unlike what we do for pages and skbs).  in gsi_trans_cmd_add()
422 	 * entry are the DMA address and length.  We still need the SG  in gsi_trans_cmd_add()
426 	sg = &trans->sgl[which];  in gsi_trans_cmd_add()
431 	trans->cmd_opcode[which] = opcode;  in gsi_trans_cmd_add()
438 	struct scatterlist *sg = &trans->sgl[0];  in gsi_trans_page_add()
441 	if (WARN_ON(trans->rsvd_count != 1))  in gsi_trans_page_add()
442 		return -EINVAL;  in gsi_trans_page_add()
443 	if (WARN_ON(trans->used_count))  in gsi_trans_page_add()
444 		return -EINVAL;  in gsi_trans_page_add()
447 	ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction);  in gsi_trans_page_add()
449 		return -ENOMEM;  in gsi_trans_page_add()
451 	trans->used_count++;	/* Transaction now owns the (DMA mapped) page */  in gsi_trans_page_add()
459 	struct scatterlist *sg = &trans->sgl[0];  in gsi_trans_skb_add()
463 	if (WARN_ON(trans->rsvd_count != 1))  in gsi_trans_skb_add()
464 		return -EINVAL;  in gsi_trans_skb_add()
465 	if (WARN_ON(trans->used_count))  in gsi_trans_skb_add()
466 		return -EINVAL;  in gsi_trans_skb_add()
468 	/* skb->len will not be 0 (checked early) */  in gsi_trans_skb_add()
469 	ret = skb_to_sgvec(skb, sg, 0, skb->len);  in gsi_trans_skb_add()
474 	ret = dma_map_sg(trans->gsi->dev, sg, used_count, trans->direction);  in gsi_trans_skb_add()
476 		return -ENOMEM;  in gsi_trans_skb_add()
478 	/* Transaction now owns the (DMA mapped) skb */  in gsi_trans_skb_add()
479 	trans->used_count += used_count;  in gsi_trans_skb_add()
532  * __gsi_trans_commit() - Common GSI transaction commit code
544 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in __gsi_trans_commit()
545 	struct gsi_ring *tre_ring = &channel->tre_ring;  in __gsi_trans_commit()
547 	bool bei = channel->toward_ipa;  in __gsi_trans_commit()
555 	WARN_ON(!trans->used_count);  in __gsi_trans_commit()
562 	cmd_opcode = channel->command ? &trans->cmd_opcode[0] : NULL;  in __gsi_trans_commit()
563 	avail = tre_ring->count - tre_ring->index % tre_ring->count;  in __gsi_trans_commit()
564 	dest_tre = gsi_ring_virt(tre_ring, tre_ring->index);  in __gsi_trans_commit()
565 	for_each_sg(trans->sgl, sg, trans->used_count, i) {  in __gsi_trans_commit()
566 		bool last_tre = i == trans->used_count - 1;  in __gsi_trans_commit()
571 		if (!avail--)  in __gsi_trans_commit()
580 	gsi_trans_map(trans, tre_ring->index);  in __gsi_trans_commit()
582 	tre_ring->index += trans->used_count;  in __gsi_trans_commit()
584 	trans->len = byte_count;  in __gsi_trans_commit()
585 	if (channel->toward_ipa)  in __gsi_trans_commit()
591 	if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) {  in __gsi_trans_commit()
593 		if (channel->toward_ipa)  in __gsi_trans_commit()
603 	if (trans->used_count)  in gsi_trans_commit()
612 	if (!trans->used_count)  in gsi_trans_commit_wait()
615 	refcount_inc(&trans->refcount);  in gsi_trans_commit_wait()
619 	wait_for_completion(&trans->completion);  in gsi_trans_commit_wait()
629 	if (trans->direction != DMA_NONE)  in gsi_trans_complete()
630 		dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used_count,  in gsi_trans_complete()
631 			     trans->direction);  in gsi_trans_complete()
635 	complete(&trans->completion);  in gsi_trans_complete()
643 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_channel_trans_cancel_pending()
644 	u16 trans_id = trans_info->pending_id;  in gsi_channel_trans_cancel_pending()
646 	/* channel->gsi->mutex is held by caller */  in gsi_channel_trans_cancel_pending()
649 	if (trans_id == trans_info->committed_id)  in gsi_channel_trans_cancel_pending()
656 		trans = &trans_info->trans[trans_id % channel->tre_count];  in gsi_channel_trans_cancel_pending()
657 		trans->cancelled = true;  in gsi_channel_trans_cancel_pending()
658 	} while (++trans_id != trans_info->committed_id);  in gsi_channel_trans_cancel_pending()
661 	trans_info->pending_id = trans_info->committed_id;  in gsi_channel_trans_cancel_pending()
664 	napi_schedule(&channel->napi);  in gsi_channel_trans_cancel_pending()
670 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_trans_read_byte()
671 	struct gsi_ring *tre_ring = &channel->tre_ring;  in gsi_trans_read_byte()
675 	trans_info = &channel->trans_info;  in gsi_trans_read_byte()
679 		return -EBUSY;  in gsi_trans_read_byte()
683 	dest_tre = gsi_ring_virt(tre_ring, tre_ring->index);  in gsi_trans_read_byte()
686 	tre_ring->index++;  in gsi_trans_read_byte()
695 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_trans_read_byte_done()
697 	gsi_trans_tre_release(&channel->trans_info, 1);  in gsi_trans_read_byte_done()
703 	struct gsi_channel *channel = &gsi->channel[channel_id];  in gsi_channel_trans_init()
704 	u32 tre_count = channel->tre_count;  in gsi_channel_trans_init()
712 	trans_info = &channel->trans_info;  in gsi_channel_trans_init()
719 	tre_max = gsi_channel_tre_max(channel->gsi, channel_id);  in gsi_channel_trans_init()
720 	atomic_set(&trans_info->tre_avail, tre_max);  in gsi_channel_trans_init()
726 	 * power-of-two number of transactions we can use an index  in gsi_channel_trans_init()
730 	trans_info->trans = kcalloc(tre_count, sizeof(*trans_info->trans),  in gsi_channel_trans_init()
732 	if (!trans_info->trans)  in gsi_channel_trans_init()
733 		return -ENOMEM;  in gsi_channel_trans_init()
734 	trans_info->free_id = 0;	/* all modulo channel->tre_count */  in gsi_channel_trans_init()
735 	trans_info->allocated_id = 0;  in gsi_channel_trans_init()
736 	trans_info->committed_id = 0;  in gsi_channel_trans_init()
737 	trans_info->pending_id = 0;  in gsi_channel_trans_init()
738 	trans_info->completed_id = 0;  in gsi_channel_trans_init()
739 	trans_info->polled_id = 0;  in gsi_channel_trans_init()
746 	trans_info->map = kcalloc(tre_count, sizeof(*trans_info->map),  in gsi_channel_trans_init()
748 	if (!trans_info->map) {  in gsi_channel_trans_init()
749 		ret = -ENOMEM;  in gsi_channel_trans_init()
759 	ret = gsi_trans_pool_init(&trans_info->sg_pool,  in gsi_channel_trans_init()
761 				  tre_max, channel->trans_tre_max);  in gsi_channel_trans_init()
769 	kfree(trans_info->map);  in gsi_channel_trans_init()
771 	kfree(trans_info->trans);  in gsi_channel_trans_init()
773 	dev_err(gsi->dev, "error %d initializing channel %u transactions\n",  in gsi_channel_trans_init()
782 	struct gsi_trans_info *trans_info = &channel->trans_info;  in gsi_channel_trans_exit()
784 	gsi_trans_pool_exit(&trans_info->sg_pool);  in gsi_channel_trans_exit()
785 	kfree(trans_info->trans);  in gsi_channel_trans_exit()
786 	kfree(trans_info->map);  in gsi_channel_trans_exit()