Lines Matching +full:map +full:- +full:to +full:- +full:dma +full:- +full:channel
1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2022 Linaro Ltd.
12 #include <linux/dma-direction.h>
24 * A GSI transaction abstracts the behavior of a GSI channel by representing
28 * by the GSI transaction core, allowing users to simply describe operations
29 * to be performed. When a transaction has completed a callback function
30 * (dependent on the type of endpoint associated with the channel) allows
33 * To perform an operation (or set of them), a user of the GSI transaction
37 * exhaustion of the available TREs in a channel ring is detected as early
38 * as possible. Any other resources that might be needed to complete a
43 * scatterlist structures are initialized by "adding" operations to the
44 * transaction. If a buffer in an operation must be mapped for DMA, this is
45 * done at the time it is added to the transaction. It is possible for a
46 * mapping error to occur when an operation is added. In this case the
50 * Once all operations have been successfully added to a transaction, the
52 * transaction to the GSI transaction core. The GSI transaction code
53 * formats the content of the scatterlist array into the channel ring
54 * buffer and informs the hardware that new TREs are available to process.
56 * The last TRE in each transaction is marked to interrupt the AP when the
59 * TRE in the transaction is sufficient to indicate the full transaction
63 * GSI code into the IPA layer, allowing it to perform any final cleanup
73 /* An entry in a channel ring */
75 __le64 addr; /* DMA address */
81 /* gsi_tre->flags mask values (in CPU byte order) */
93 return -EINVAL; in gsi_trans_pool_init()
95 return -EINVAL; in gsi_trans_pool_init()
97 return -EINVAL; in gsi_trans_pool_init()
106 virt = kcalloc(count + max_alloc - 1, size, GFP_KERNEL); in gsi_trans_pool_init()
108 return -ENOMEM; in gsi_trans_pool_init()
110 pool->base = virt; in gsi_trans_pool_init()
112 pool->count = ksize(pool->base) / size; in gsi_trans_pool_init()
113 pool->free = 0; in gsi_trans_pool_init()
114 pool->max_alloc = max_alloc; in gsi_trans_pool_init()
115 pool->size = size; in gsi_trans_pool_init()
116 pool->addr = 0; /* Only used for DMA pools */ in gsi_trans_pool_init()
123 kfree(pool->base); in gsi_trans_pool_exit()
127 /* Home-grown DMA pool. This way we can preallocate the pool, and guarantee
129 * require up to max_alloc elements from the pool. But we only allow
130 * allocation of a single element from a DMA pool at a time.
140 return -EINVAL; in gsi_trans_pool_init_dma()
142 return -EINVAL; in gsi_trans_pool_init_dma()
144 return -EINVAL; in gsi_trans_pool_init_dma()
146 /* Don't let allocations cross a power-of-two boundary */ in gsi_trans_pool_init_dma()
148 total_size = (count + max_alloc - 1) * size; in gsi_trans_pool_init_dma()
150 /* The allocator will give us a power-of-2 number of pages in gsi_trans_pool_init_dma()
151 * sufficient to satisfy our request. Round up our requested in gsi_trans_pool_init_dma()
152 * size to avoid any unused space in the allocation. This way in gsi_trans_pool_init_dma()
160 return -ENOMEM; in gsi_trans_pool_init_dma()
162 pool->base = virt; in gsi_trans_pool_init_dma()
163 pool->count = total_size / size; in gsi_trans_pool_init_dma()
164 pool->free = 0; in gsi_trans_pool_init_dma()
165 pool->size = size; in gsi_trans_pool_init_dma()
166 pool->max_alloc = max_alloc; in gsi_trans_pool_init_dma()
167 pool->addr = addr; in gsi_trans_pool_init_dma()
174 size_t total_size = pool->count * pool->size; in gsi_trans_pool_exit_dma()
176 dma_free_coherent(dev, total_size, pool->base, pool->addr); in gsi_trans_pool_exit_dma()
186 WARN_ON(count > pool->max_alloc); in gsi_trans_pool_alloc_common()
189 if (count > pool->count - pool->free) in gsi_trans_pool_alloc_common()
190 pool->free = 0; in gsi_trans_pool_alloc_common()
192 offset = pool->free * pool->size; in gsi_trans_pool_alloc_common()
193 pool->free += count; in gsi_trans_pool_alloc_common()
194 memset(pool->base + offset, 0, count * pool->size); in gsi_trans_pool_alloc_common()
202 return pool->base + gsi_trans_pool_alloc_common(pool, count); in gsi_trans_pool_alloc()
205 /* Allocate a single zeroed entry from a DMA pool */
210 *addr = pool->addr + offset; in gsi_trans_pool_alloc_dma()
212 return pool->base + offset; in gsi_trans_pool_alloc_dma()
215 /* Map a TRE ring entry index to the transaction it is associated with */
218 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_map() local
221 index += trans->used_count - 1; in gsi_trans_map()
224 channel->trans_info.map[index % channel->tre_ring.count] = trans; in gsi_trans_map()
227 /* Return the transaction mapped to a given ring entry */
229 gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index) in gsi_channel_trans_mapped() argument
232 return channel->trans_info.map[index % channel->tre_ring.count]; in gsi_channel_trans_mapped()
235 /* Return the oldest completed transaction for a channel (or null) */
236 struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) in gsi_channel_trans_complete() argument
238 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_complete()
239 u16 trans_id = trans_info->completed_id; in gsi_channel_trans_complete()
241 if (trans_id == trans_info->pending_id) { in gsi_channel_trans_complete()
242 gsi_channel_update(channel); in gsi_channel_trans_complete()
243 if (trans_id == trans_info->pending_id) in gsi_channel_trans_complete()
247 return &trans_info->trans[trans_id %= channel->tre_count]; in gsi_channel_trans_complete()
250 /* Move a transaction from allocated to committed state */
253 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_committed() local
254 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_committed()
257 trans_info->allocated_id++; in gsi_trans_move_committed()
260 /* Move committed transactions to pending state */
263 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_pending() local
264 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_pending()
265 u16 trans_index = trans - &trans_info->trans[0]; in gsi_trans_move_pending()
269 delta = trans_index - trans_info->committed_id + 1; in gsi_trans_move_pending()
270 trans_info->committed_id += delta % channel->tre_count; in gsi_trans_move_pending()
273 /* Move pending transactions to completed state */
276 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_complete() local
277 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_complete()
278 u16 trans_index = trans - trans_info->trans; in gsi_trans_move_complete()
282 delta = trans_index - trans_info->pending_id + 1; in gsi_trans_move_complete()
283 delta %= channel->tre_count; in gsi_trans_move_complete()
284 trans_info->pending_id += delta; in gsi_trans_move_complete()
287 /* Move a transaction from completed to polled state */
290 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_polled() local
291 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_polled()
294 trans_info->completed_id++; in gsi_trans_move_polled()
297 /* Reserve some number of TREs on a channel. Returns true if successful */
301 int avail = atomic_read(&trans_info->tre_avail); in gsi_trans_tre_reserve()
305 new = avail - (int)tre_count; in gsi_trans_tre_reserve()
308 } while (!atomic_try_cmpxchg(&trans_info->tre_avail, &avail, new)); in gsi_trans_tre_reserve()
313 /* Release previously-reserved TRE entries to a channel */
317 atomic_add(tre_count, &trans_info->tre_avail); in gsi_trans_tre_release()
326 trans_info = &gsi->channel[channel_id].trans_info; in gsi_channel_trans_idle()
328 return atomic_read(&trans_info->tre_avail) == tre_max; in gsi_channel_trans_idle()
331 /* Allocate a GSI transaction on a channel */
336 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_alloc() local
341 if (WARN_ON(tre_count > channel->trans_tre_max)) in gsi_channel_trans_alloc()
344 trans_info = &channel->trans_info; in gsi_channel_trans_alloc()
350 trans_index = trans_info->free_id % channel->tre_count; in gsi_channel_trans_alloc()
351 trans = &trans_info->trans[trans_index]; in gsi_channel_trans_alloc()
354 /* Initialize non-zero fields in the transaction */ in gsi_channel_trans_alloc()
355 trans->gsi = gsi; in gsi_channel_trans_alloc()
356 trans->channel_id = channel_id; in gsi_channel_trans_alloc()
357 trans->rsvd_count = tre_count; in gsi_channel_trans_alloc()
358 init_completion(&trans->completion); in gsi_channel_trans_alloc()
361 trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count); in gsi_channel_trans_alloc()
362 sg_init_marker(trans->sgl, tre_count); in gsi_channel_trans_alloc()
364 trans->direction = direction; in gsi_channel_trans_alloc()
365 refcount_set(&trans->refcount, 1); in gsi_channel_trans_alloc()
368 trans_info->free_id++; in gsi_channel_trans_alloc()
373 /* Free a previously-allocated transaction */
378 if (!refcount_dec_and_test(&trans->refcount)) in gsi_trans_free()
384 trans_info = &trans->gsi->channel[trans->channel_id].trans_info; in gsi_trans_free()
385 if (!trans->used_count) { in gsi_trans_free()
386 trans_info->allocated_id++; in gsi_trans_free()
387 trans_info->committed_id++; in gsi_trans_free()
388 trans_info->pending_id++; in gsi_trans_free()
389 trans_info->completed_id++; in gsi_trans_free()
395 trans_info->polled_id++; in gsi_trans_free()
400 gsi_trans_tre_release(trans_info, trans->rsvd_count); in gsi_trans_free()
403 /* Add an immediate command to a transaction */
407 u32 which = trans->used_count++; in gsi_trans_cmd_add()
410 WARN_ON(which >= trans->rsvd_count); in gsi_trans_cmd_add()
414 * using dma_alloc_coherent(). We therefore do *not* map them in gsi_trans_cmd_add()
415 * for DMA (unlike what we do for pages and skbs). in gsi_trans_cmd_add()
419 * gsi_trans_complete() to skip the unmapping step. in gsi_trans_cmd_add()
422 * entry are the DMA address and length. We still need the SG in gsi_trans_cmd_add()
423 * table flags to be maintained though, so assign a NULL page in gsi_trans_cmd_add()
426 sg = &trans->sgl[which]; in gsi_trans_cmd_add()
431 trans->cmd_opcode[which] = opcode; in gsi_trans_cmd_add()
434 /* Add a page transfer to a transaction. It will fill the only TRE. */
438 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_page_add()
441 if (WARN_ON(trans->rsvd_count != 1)) in gsi_trans_page_add()
442 return -EINVAL; in gsi_trans_page_add()
443 if (WARN_ON(trans->used_count)) in gsi_trans_page_add()
444 return -EINVAL; in gsi_trans_page_add()
447 ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction); in gsi_trans_page_add()
449 return -ENOMEM; in gsi_trans_page_add()
451 trans->used_count++; /* Transaction now owns the (DMA mapped) page */ in gsi_trans_page_add()
456 /* Add an SKB transfer to a transaction. No other TREs will be used. */
459 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_skb_add()
463 if (WARN_ON(trans->rsvd_count != 1)) in gsi_trans_skb_add()
464 return -EINVAL; in gsi_trans_skb_add()
465 if (WARN_ON(trans->used_count)) in gsi_trans_skb_add()
466 return -EINVAL; in gsi_trans_skb_add()
468 /* skb->len will not be 0 (checked early) */ in gsi_trans_skb_add()
469 ret = skb_to_sgvec(skb, sg, 0, skb->len); in gsi_trans_skb_add()
474 ret = dma_map_sg(trans->gsi->dev, sg, used_count, trans->direction); in gsi_trans_skb_add()
476 return -ENOMEM; in gsi_trans_skb_add()
478 /* Transaction now owns the (DMA mapped) skb */ in gsi_trans_skb_add()
479 trans->used_count += used_count; in gsi_trans_skb_add()
484 /* Compute the length/opcode value to use for a TRE */
491 /* Compute the flags value to use for a given TRE */
507 } else { /* All others indicate there's more to come */ in gsi_tre_flags()
526 * Doing the assignment this way is an attempt to make that happen. in gsi_trans_tre_fill()
532 * __gsi_trans_commit() - Common GSI transaction commit code
533 * @trans: Transaction to commit
534 * @ring_db: Whether to tell the hardware about these queued transfers
536 * Formats channel ring TRE entries based on the content of the scatterlist.
537 * Maps a transaction pointer to the last ring entry used for the transaction,
538 * so it can be recovered when it completes. Moves the transaction to
539 * pending state. Finally, updates the channel ring pointer and optionally
544 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in __gsi_trans_commit() local
545 struct gsi_ring *tre_ring = &channel->tre_ring; in __gsi_trans_commit()
547 bool bei = channel->toward_ipa; in __gsi_trans_commit()
555 WARN_ON(!trans->used_count); in __gsi_trans_commit()
558 * filling them we'll switch to the beginning to finish. in __gsi_trans_commit()
562 cmd_opcode = channel->command ? &trans->cmd_opcode[0] : NULL; in __gsi_trans_commit()
563 avail = tre_ring->count - tre_ring->index % tre_ring->count; in __gsi_trans_commit()
564 dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); in __gsi_trans_commit()
565 for_each_sg(trans->sgl, sg, trans->used_count, i) { in __gsi_trans_commit()
566 bool last_tre = i == trans->used_count - 1; in __gsi_trans_commit()
571 if (!avail--) in __gsi_trans_commit()
580 gsi_trans_map(trans, tre_ring->index); in __gsi_trans_commit()
582 tre_ring->index += trans->used_count; in __gsi_trans_commit()
584 trans->len = byte_count; in __gsi_trans_commit()
585 if (channel->toward_ipa) in __gsi_trans_commit()
591 if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) { in __gsi_trans_commit()
592 /* Report what we're handing off to hardware for TX channels */ in __gsi_trans_commit()
593 if (channel->toward_ipa) in __gsi_trans_commit()
596 gsi_channel_doorbell(channel); in __gsi_trans_commit()
603 if (trans->used_count) in gsi_trans_commit()
609 /* Commit a GSI transaction and wait for it to complete */
612 if (!trans->used_count) in gsi_trans_commit_wait()
615 refcount_inc(&trans->refcount); in gsi_trans_commit_wait()
619 wait_for_completion(&trans->completion); in gsi_trans_commit_wait()
629 if (trans->direction != DMA_NONE) in gsi_trans_complete()
630 dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used_count, in gsi_trans_complete()
631 trans->direction); in gsi_trans_complete()
635 complete(&trans->completion); in gsi_trans_complete()
640 /* Cancel a channel's pending transactions */
641 void gsi_channel_trans_cancel_pending(struct gsi_channel *channel) in gsi_channel_trans_cancel_pending() argument
643 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_cancel_pending()
644 u16 trans_id = trans_info->pending_id; in gsi_channel_trans_cancel_pending()
646 /* channel->gsi->mutex is held by caller */ in gsi_channel_trans_cancel_pending()
649 if (trans_id == trans_info->committed_id) in gsi_channel_trans_cancel_pending()
656 trans = &trans_info->trans[trans_id % channel->tre_count]; in gsi_channel_trans_cancel_pending()
657 trans->cancelled = true; in gsi_channel_trans_cancel_pending()
658 } while (++trans_id != trans_info->committed_id); in gsi_channel_trans_cancel_pending()
661 trans_info->pending_id = trans_info->committed_id; in gsi_channel_trans_cancel_pending()
663 /* Schedule NAPI polling to complete the cancelled transactions */ in gsi_channel_trans_cancel_pending()
664 napi_schedule(&channel->napi); in gsi_channel_trans_cancel_pending()
667 /* Issue a command to read a single byte from a channel */
670 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte() local
671 struct gsi_ring *tre_ring = &channel->tre_ring; in gsi_trans_read_byte()
675 trans_info = &channel->trans_info; in gsi_trans_read_byte()
679 return -EBUSY; in gsi_trans_read_byte()
683 dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); in gsi_trans_read_byte()
686 tre_ring->index++; in gsi_trans_read_byte()
687 gsi_channel_doorbell(channel); in gsi_trans_read_byte()
695 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte_done() local
697 gsi_trans_tre_release(&channel->trans_info, 1); in gsi_trans_read_byte_done()
700 /* Initialize a channel's GSI transaction info */
703 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_init() local
704 u32 tre_count = channel->tre_count; in gsi_channel_trans_init()
709 /* Ensure the size of a channel element is what's expected */ in gsi_channel_trans_init()
712 trans_info = &channel->trans_info; in gsi_channel_trans_init()
719 tre_max = gsi_channel_tre_max(channel->gsi, channel_id); in gsi_channel_trans_init()
720 atomic_set(&trans_info->tre_avail, tre_max); in gsi_channel_trans_init()
725 * it to something a little less than that). By allocating a in gsi_channel_trans_init()
726 * power-of-two number of transactions we can use an index in gsi_channel_trans_init()
727 * modulo that number to determine the next one that's free. in gsi_channel_trans_init()
730 trans_info->trans = kcalloc(tre_count, sizeof(*trans_info->trans), in gsi_channel_trans_init()
732 if (!trans_info->trans) in gsi_channel_trans_init()
733 return -ENOMEM; in gsi_channel_trans_init()
734 trans_info->free_id = 0; /* all modulo channel->tre_count */ in gsi_channel_trans_init()
735 trans_info->allocated_id = 0; in gsi_channel_trans_init()
736 trans_info->committed_id = 0; in gsi_channel_trans_init()
737 trans_info->pending_id = 0; in gsi_channel_trans_init()
738 trans_info->completed_id = 0; in gsi_channel_trans_init()
739 trans_info->polled_id = 0; in gsi_channel_trans_init()
741 /* A completion event contains a pointer to the TRE that caused in gsi_channel_trans_init()
743 * Each entry in this map records the transaction associated in gsi_channel_trans_init()
746 trans_info->map = kcalloc(tre_count, sizeof(*trans_info->map), in gsi_channel_trans_init()
748 if (!trans_info->map) { in gsi_channel_trans_init()
749 ret = -ENOMEM; in gsi_channel_trans_init()
753 /* A transaction uses a scatterlist array to represent the data in gsi_channel_trans_init()
755 * element is used to fill a single TRE when the transaction is in gsi_channel_trans_init()
759 ret = gsi_trans_pool_init(&trans_info->sg_pool, in gsi_channel_trans_init()
761 tre_max, channel->trans_tre_max); in gsi_channel_trans_init()
769 kfree(trans_info->map); in gsi_channel_trans_init()
771 kfree(trans_info->trans); in gsi_channel_trans_init()
773 dev_err(gsi->dev, "error %d initializing channel %u transactions\n", in gsi_channel_trans_init()
780 void gsi_channel_trans_exit(struct gsi_channel *channel) in gsi_channel_trans_exit() argument
782 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_exit()
784 gsi_trans_pool_exit(&trans_info->sg_pool); in gsi_channel_trans_exit()
785 kfree(trans_info->trans); in gsi_channel_trans_exit()
786 kfree(trans_info->map); in gsi_channel_trans_exit()