Lines Matching +full:map +full:- +full:to +full:- +full:dma +full:- +full:channel
1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2020 Linaro Ltd.
12 #include <linux/dma-direction.h>
24 * A GSI transaction abstracts the behavior of a GSI channel by representing
28 * by the GSI transaction core, allowing users to simply describe commands
29 * to be performed. When a transaction has completed a callback function
30 * (dependent on the type of endpoint associated with the channel) allows
33 * To perform a command (or set of them), a user of the GSI transaction
37 * exhaustion of the available TREs in a channel ring is detected
38 * as early as possible. All resources required to complete a transaction
48 * for DMA if necessary, and this should be done *before* allocating
52 * Committing transfers ownership of the entire transaction to the GSI
54 * the scatterlist array into the channel ring buffer and informs the
55 * hardware that new TREs are available to process.
57 * The last TRE in each transaction is marked to interrupt the AP when the
60 * TRE in the transaction is sufficient to indicate the full transaction
64 * GSI code into the IPA layer, allowing it to perform any final cleanup
74 /* An entry in a channel ring */
76 __le64 addr; /* DMA address */
82 /* gsi_tre->flags mask values (in CPU byte order) */
95 return -EINVAL; in gsi_trans_pool_init()
97 return -EINVAL; in gsi_trans_pool_init()
99 return -EINVAL; in gsi_trans_pool_init()
109 virt = kcalloc(count + max_alloc - 1, size, GFP_KERNEL); in gsi_trans_pool_init()
111 return -ENOMEM; in gsi_trans_pool_init()
113 pool->base = virt; in gsi_trans_pool_init()
115 pool->count = ksize(pool->base) / size; in gsi_trans_pool_init()
116 pool->free = 0; in gsi_trans_pool_init()
117 pool->max_alloc = max_alloc; in gsi_trans_pool_init()
118 pool->size = size; in gsi_trans_pool_init()
119 pool->addr = 0; /* Only used for DMA pools */ in gsi_trans_pool_init()
126 kfree(pool->base); in gsi_trans_pool_exit()
131 /* Home-grown DMA pool. This way we can preallocate and use the tre_count
132 * to guarantee allocations will succeed. Even though we specify max_alloc
134 * element from a DMA pool.
145 return -EINVAL; in gsi_trans_pool_init_dma()
147 return -EINVAL; in gsi_trans_pool_init_dma()
149 return -EINVAL; in gsi_trans_pool_init_dma()
152 /* Don't let allocations cross a power-of-two boundary */ in gsi_trans_pool_init_dma()
154 total_size = (count + max_alloc - 1) * size; in gsi_trans_pool_init_dma()
156 /* The allocator will give us a power-of-2 number of pages. But we in gsi_trans_pool_init_dma()
167 return -ENOMEM; in gsi_trans_pool_init_dma()
169 pool->base = virt; in gsi_trans_pool_init_dma()
170 pool->count = total_size / size; in gsi_trans_pool_init_dma()
171 pool->free = 0; in gsi_trans_pool_init_dma()
172 pool->size = size; in gsi_trans_pool_init_dma()
173 pool->max_alloc = max_alloc; in gsi_trans_pool_init_dma()
174 pool->addr = addr; in gsi_trans_pool_init_dma()
181 size_t total_size = pool->count * pool->size; in gsi_trans_pool_exit_dma()
183 dma_free_coherent(dev, total_size, pool->base, pool->addr); in gsi_trans_pool_exit_dma()
193 /* assert(count <= pool->max_alloc); */ in gsi_trans_pool_alloc_common()
196 if (count > pool->count - pool->free) in gsi_trans_pool_alloc_common()
197 pool->free = 0; in gsi_trans_pool_alloc_common()
199 offset = pool->free * pool->size; in gsi_trans_pool_alloc_common()
200 pool->free += count; in gsi_trans_pool_alloc_common()
201 memset(pool->base + offset, 0, count * pool->size); in gsi_trans_pool_alloc_common()
209 return pool->base + gsi_trans_pool_alloc_common(pool, count); in gsi_trans_pool_alloc()
212 /* Allocate a single zeroed entry from a DMA pool */
217 *addr = pool->addr + offset; in gsi_trans_pool_alloc_dma()
219 return pool->base + offset; in gsi_trans_pool_alloc_dma()
227 void *end = pool->base + pool->count * pool->size; in gsi_trans_pool_next()
229 /* assert(element >= pool->base); */ in gsi_trans_pool_next()
231 /* assert(pool->max_alloc == 1); */ in gsi_trans_pool_next()
232 element += pool->size; in gsi_trans_pool_next()
234 return element < end ? element : pool->base; in gsi_trans_pool_next()
237 /* Map a given ring entry index to the transaction associated with it */
238 static void gsi_channel_trans_map(struct gsi_channel *channel, u32 index, in gsi_channel_trans_map() argument
242 channel->trans_info.map[index % channel->tre_ring.count] = trans; in gsi_channel_trans_map()
245 /* Return the transaction mapped to a given ring entry */
247 gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index) in gsi_channel_trans_mapped() argument
250 return channel->trans_info.map[index % channel->tre_ring.count]; in gsi_channel_trans_mapped()
253 /* Return the oldest completed transaction for a channel (or null) */
254 struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) in gsi_channel_trans_complete() argument
256 return list_first_entry_or_null(&channel->trans_info.complete, in gsi_channel_trans_complete()
260 /* Move a transaction from the allocated list to the pending list */
263 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_pending() local
264 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_pending()
266 spin_lock_bh(&trans_info->spinlock); in gsi_trans_move_pending()
268 list_move_tail(&trans->links, &trans_info->pending); in gsi_trans_move_pending()
270 spin_unlock_bh(&trans_info->spinlock); in gsi_trans_move_pending()
274 * to the completed list.
278 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_complete() local
279 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_complete()
282 spin_lock_bh(&trans_info->spinlock); in gsi_trans_move_complete()
284 /* Move this transaction and all predecessors to completed list */ in gsi_trans_move_complete()
285 list_cut_position(&list, &trans_info->pending, &trans->links); in gsi_trans_move_complete()
286 list_splice_tail(&list, &trans_info->complete); in gsi_trans_move_complete()
288 spin_unlock_bh(&trans_info->spinlock); in gsi_trans_move_complete()
291 /* Move a transaction from the completed list to the polled list */
294 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_polled() local
295 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_polled()
297 spin_lock_bh(&trans_info->spinlock); in gsi_trans_move_polled()
299 list_move_tail(&trans->links, &trans_info->polled); in gsi_trans_move_polled()
301 spin_unlock_bh(&trans_info->spinlock); in gsi_trans_move_polled()
304 /* Reserve some number of TREs on a channel. Returns true if successful */
308 int avail = atomic_read(&trans_info->tre_avail); in gsi_trans_tre_reserve()
312 new = avail - (int)tre_count; in gsi_trans_tre_reserve()
315 } while (!atomic_try_cmpxchg(&trans_info->tre_avail, &avail, new)); in gsi_trans_tre_reserve()
320 /* Release previously-reserved TRE entries to a channel */
324 atomic_add(tre_count, &trans_info->tre_avail); in gsi_trans_tre_release()
327 /* Allocate a GSI transaction on a channel */
332 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_alloc() local
338 trans_info = &channel->trans_info; in gsi_channel_trans_alloc()
346 /* Allocate and initialize non-zero fields in the the transaction */ in gsi_channel_trans_alloc()
347 trans = gsi_trans_pool_alloc(&trans_info->pool, 1); in gsi_channel_trans_alloc()
348 trans->gsi = gsi; in gsi_channel_trans_alloc()
349 trans->channel_id = channel_id; in gsi_channel_trans_alloc()
350 trans->tre_count = tre_count; in gsi_channel_trans_alloc()
351 init_completion(&trans->completion); in gsi_channel_trans_alloc()
354 trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count); in gsi_channel_trans_alloc()
355 sg_init_marker(trans->sgl, tre_count); in gsi_channel_trans_alloc()
357 trans->direction = direction; in gsi_channel_trans_alloc()
359 spin_lock_bh(&trans_info->spinlock); in gsi_channel_trans_alloc()
361 list_add_tail(&trans->links, &trans_info->alloc); in gsi_channel_trans_alloc()
363 spin_unlock_bh(&trans_info->spinlock); in gsi_channel_trans_alloc()
365 refcount_set(&trans->refcount, 1); in gsi_channel_trans_alloc()
370 /* Free a previously-allocated transaction */
373 refcount_t *refcount = &trans->refcount; in gsi_trans_free()
377 /* We must hold the lock to release the last reference */ in gsi_trans_free()
381 trans_info = &trans->gsi->channel[trans->channel_id].trans_info; in gsi_trans_free()
383 spin_lock_bh(&trans_info->spinlock); in gsi_trans_free()
388 list_del(&trans->links); in gsi_trans_free()
390 spin_unlock_bh(&trans_info->spinlock); in gsi_trans_free()
400 gsi_trans_tre_release(trans_info, trans->tre_count); in gsi_trans_free()
403 /* Add an immediate command to a transaction */
409 u32 which = trans->used++; in gsi_trans_cmd_add()
412 /* assert(which < trans->tre_count); */ in gsi_trans_cmd_add()
416 * using dma_alloc_coherent(). We therefore do *not* map them in gsi_trans_cmd_add()
417 * for DMA (unlike what we do for pages and skbs). in gsi_trans_cmd_add()
421 * gsi_trans_complete() to skip the unmapping step. in gsi_trans_cmd_add()
424 * entry are the DMA address and length. We still need the SG in gsi_trans_cmd_add()
425 * table flags to be maintained though, so assign a NULL page in gsi_trans_cmd_add()
428 sg = &trans->sgl[which]; in gsi_trans_cmd_add()
433 info = &trans->info[which]; in gsi_trans_cmd_add()
434 info->opcode = opcode; in gsi_trans_cmd_add()
435 info->direction = direction; in gsi_trans_cmd_add()
438 /* Add a page transfer to a transaction. It will fill the only TRE. */
442 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_page_add()
445 /* assert(trans->tre_count == 1); */ in gsi_trans_page_add()
446 /* assert(!trans->used); */ in gsi_trans_page_add()
449 ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction); in gsi_trans_page_add()
451 return -ENOMEM; in gsi_trans_page_add()
453 trans->used++; /* Transaction now owns the (DMA mapped) page */ in gsi_trans_page_add()
458 /* Add an SKB transfer to a transaction. No other TREs will be used. */
461 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_skb_add()
465 /* assert(trans->tre_count == 1); */ in gsi_trans_skb_add()
466 /* assert(!trans->used); */ in gsi_trans_skb_add()
468 /* skb->len will not be 0 (checked early) */ in gsi_trans_skb_add()
469 ret = skb_to_sgvec(skb, sg, 0, skb->len); in gsi_trans_skb_add()
474 ret = dma_map_sg(trans->gsi->dev, sg, used, trans->direction); in gsi_trans_skb_add()
476 return -ENOMEM; in gsi_trans_skb_add()
478 trans->used += used; /* Transaction now owns the (DMA mapped) skb */ in gsi_trans_skb_add()
483 /* Compute the length/opcode value to use for a TRE */
490 /* Compute the flags value to use for a given TRE */
506 } else { /* All others indicate there's more to come */ in gsi_tre_flags()
525 * Doing the assignment this way is an attempt to make that happen. in gsi_trans_tre_fill()
531 * __gsi_trans_commit() - Common GSI transaction commit code
532 * @trans: Transaction to commit
533 * @ring_db: Whether to tell the hardware about these queued transfers
535 * Formats channel ring TRE entries based on the content of the scatterlist.
536 * Maps a transaction pointer to the last ring entry used for the transaction,
537 * so it can be recovered when it completes. Moves the transaction to the
538 * pending list. Finally, updates the channel ring pointer and optionally
543 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in __gsi_trans_commit() local
544 struct gsi_ring *ring = &channel->tre_ring; in __gsi_trans_commit()
546 bool bei = channel->toward_ipa; in __gsi_trans_commit()
554 /* assert(trans->used > 0); */ in __gsi_trans_commit()
557 * filling them we'll switch to the beginning to finish. in __gsi_trans_commit()
561 info = trans->info ? &trans->info[0] : NULL; in __gsi_trans_commit()
562 avail = ring->count - ring->index % ring->count; in __gsi_trans_commit()
563 dest_tre = gsi_ring_virt(ring, ring->index); in __gsi_trans_commit()
564 for_each_sg(trans->sgl, sg, trans->used, i) { in __gsi_trans_commit()
565 bool last_tre = i == trans->used - 1; in __gsi_trans_commit()
570 if (!avail--) in __gsi_trans_commit()
573 opcode = info++->opcode; in __gsi_trans_commit()
578 ring->index += trans->used; in __gsi_trans_commit()
580 if (channel->toward_ipa) { in __gsi_trans_commit()
582 trans->len = byte_count; in __gsi_trans_commit()
583 trans->trans_count = channel->trans_count; in __gsi_trans_commit()
584 trans->byte_count = channel->byte_count; in __gsi_trans_commit()
585 channel->trans_count++; in __gsi_trans_commit()
586 channel->byte_count += byte_count; in __gsi_trans_commit()
590 gsi_channel_trans_map(channel, ring->index - 1, trans); in __gsi_trans_commit()
595 if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) { in __gsi_trans_commit()
596 /* Report what we're handing off to hardware for TX channels */ in __gsi_trans_commit()
597 if (channel->toward_ipa) in __gsi_trans_commit()
598 gsi_channel_tx_queued(channel); in __gsi_trans_commit()
599 gsi_channel_doorbell(channel); in __gsi_trans_commit()
606 if (trans->used) in gsi_trans_commit()
612 /* Commit a GSI transaction and wait for it to complete */
615 if (!trans->used) in gsi_trans_commit_wait()
618 refcount_inc(&trans->refcount); in gsi_trans_commit_wait()
622 wait_for_completion(&trans->completion); in gsi_trans_commit_wait()
628 /* Commit a GSI transaction and wait for it to complete, with timeout */
635 if (!trans->used) in gsi_trans_commit_wait_timeout()
638 refcount_inc(&trans->refcount); in gsi_trans_commit_wait_timeout()
642 remaining = wait_for_completion_timeout(&trans->completion, in gsi_trans_commit_wait_timeout()
647 return remaining ? 0 : -ETIMEDOUT; in gsi_trans_commit_wait_timeout()
654 if (trans->direction != DMA_NONE) in gsi_trans_complete()
655 dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used, in gsi_trans_complete()
656 trans->direction); in gsi_trans_complete()
660 complete(&trans->completion); in gsi_trans_complete()
665 /* Cancel a channel's pending transactions */
666 void gsi_channel_trans_cancel_pending(struct gsi_channel *channel) in gsi_channel_trans_cancel_pending() argument
668 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_cancel_pending()
672 /* channel->gsi->mutex is held by caller */ in gsi_channel_trans_cancel_pending()
673 spin_lock_bh(&trans_info->spinlock); in gsi_channel_trans_cancel_pending()
675 cancelled = !list_empty(&trans_info->pending); in gsi_channel_trans_cancel_pending()
676 list_for_each_entry(trans, &trans_info->pending, links) in gsi_channel_trans_cancel_pending()
677 trans->cancelled = true; in gsi_channel_trans_cancel_pending()
679 list_splice_tail_init(&trans_info->pending, &trans_info->complete); in gsi_channel_trans_cancel_pending()
681 spin_unlock_bh(&trans_info->spinlock); in gsi_channel_trans_cancel_pending()
683 /* Schedule NAPI polling to complete the cancelled transactions */ in gsi_channel_trans_cancel_pending()
685 napi_schedule(&channel->napi); in gsi_channel_trans_cancel_pending()
688 /* Issue a command to read a single byte from a channel */
691 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte() local
692 struct gsi_ring *ring = &channel->tre_ring; in gsi_trans_read_byte()
696 trans_info = &channel->trans_info; in gsi_trans_read_byte()
700 return -EBUSY; in gsi_trans_read_byte()
704 dest_tre = gsi_ring_virt(ring, ring->index); in gsi_trans_read_byte()
707 ring->index++; in gsi_trans_read_byte()
708 gsi_channel_doorbell(channel); in gsi_trans_read_byte()
716 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte_done() local
718 gsi_trans_tre_release(&channel->trans_info, 1); in gsi_trans_read_byte_done()
721 /* Initialize a channel's GSI transaction info */
724 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_init() local
729 /* Ensure the size of a channel element is what's expected */ in gsi_channel_trans_init()
732 /* The map array is used to determine what transaction is associated in gsi_channel_trans_init()
734 * map entry per TRE. in gsi_channel_trans_init()
736 trans_info = &channel->trans_info; in gsi_channel_trans_init()
737 trans_info->map = kcalloc(channel->tre_count, sizeof(*trans_info->map), in gsi_channel_trans_init()
739 if (!trans_info->map) in gsi_channel_trans_init()
740 return -ENOMEM; in gsi_channel_trans_init()
745 * it to something a little less than that). We allocate resources in gsi_channel_trans_init()
749 tre_max = gsi_channel_tre_max(channel->gsi, channel_id); in gsi_channel_trans_init()
752 ret = gsi_trans_pool_init(&trans_info->pool, sizeof(struct gsi_trans), in gsi_channel_trans_init()
757 /* A transaction uses a scatterlist array to represent the data in gsi_channel_trans_init()
759 * element is used to fill a single TRE when the transaction is in gsi_channel_trans_init()
763 * All TREs in a transaction must fit within the channel's TLV FIFO. in gsi_channel_trans_init()
764 * A transaction on a channel can allocate as many TREs as that but in gsi_channel_trans_init()
767 ret = gsi_trans_pool_init(&trans_info->sg_pool, in gsi_channel_trans_init()
769 tre_max, channel->tlv_count); in gsi_channel_trans_init()
780 atomic_set(&trans_info->tre_avail, tre_max); in gsi_channel_trans_init()
782 spin_lock_init(&trans_info->spinlock); in gsi_channel_trans_init()
783 INIT_LIST_HEAD(&trans_info->alloc); in gsi_channel_trans_init()
784 INIT_LIST_HEAD(&trans_info->pending); in gsi_channel_trans_init()
785 INIT_LIST_HEAD(&trans_info->complete); in gsi_channel_trans_init()
786 INIT_LIST_HEAD(&trans_info->polled); in gsi_channel_trans_init()
791 gsi_trans_pool_exit(&trans_info->pool); in gsi_channel_trans_init()
793 kfree(trans_info->map); in gsi_channel_trans_init()
795 dev_err(gsi->dev, "error %d initializing channel %u transactions\n", in gsi_channel_trans_init()
802 void gsi_channel_trans_exit(struct gsi_channel *channel) in gsi_channel_trans_exit() argument
804 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_exit()
806 gsi_trans_pool_exit(&trans_info->sg_pool); in gsi_channel_trans_exit()
807 gsi_trans_pool_exit(&trans_info->pool); in gsi_channel_trans_exit()
808 kfree(trans_info->map); in gsi_channel_trans_exit()