Lines Matching +full:iommu +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0
19 #include <linux/dma-map-ops.h>
20 #include <asm/iommu-common.h>
22 #include <asm/iommu.h>
57 unsigned long prot; /* IOMMU page protections */
71 p->dev = dev; in iommu_batch_start()
72 p->prot = prot; in iommu_batch_start()
73 p->entry = entry; in iommu_batch_start()
74 p->npages = 0; in iommu_batch_start()
77 static inline bool iommu_use_atu(struct iommu *iommu, u64 mask) in iommu_use_atu() argument
79 return iommu->atu && mask > DMA_BIT_MASK(32); in iommu_use_atu()
85 struct pci_pbm_info *pbm = p->dev->archdata.host_controller; in iommu_batch_flush()
86 u64 *pglist = p->pglist; in iommu_batch_flush()
88 unsigned long devhandle = pbm->devhandle; in iommu_batch_flush()
89 unsigned long prot = p->prot; in iommu_batch_flush()
90 unsigned long entry = p->entry; in iommu_batch_flush()
91 unsigned long npages = p->npages; in iommu_batch_flush()
101 if (!iommu_use_atu(pbm->iommu, mask)) { in iommu_batch_flush()
108 pr_err_ratelimited("%s: IOMMU map of [%08lx:%08llx:%lx:%lx:%lx] failed with status %ld\n", in iommu_batch_flush()
114 return -1; in iommu_batch_flush()
118 iotsb_num = pbm->iommu->atu->iotsb->iotsb_num; in iommu_batch_flush()
126 pr_err_ratelimited("%s: ATU map of [%08lx:%lx:%llx:%lx:%lx] failed with status %ld\n", in iommu_batch_flush()
131 return -1; in iommu_batch_flush()
135 npages -= num; in iommu_batch_flush()
139 p->entry = entry; in iommu_batch_flush()
140 p->npages = 0; in iommu_batch_flush()
149 if (p->entry + p->npages == entry) in iommu_batch_new_entry()
151 if (p->entry != ~0UL) in iommu_batch_new_entry()
153 p->entry = entry; in iommu_batch_new_entry()
161 BUG_ON(p->npages >= PGLIST_NENTS); in iommu_batch_add()
163 p->pglist[p->npages++] = phys_page; in iommu_batch_add()
164 if (p->npages == PGLIST_NENTS) in iommu_batch_add()
175 BUG_ON(p->npages >= PGLIST_NENTS); in iommu_batch_end()
187 struct iommu *iommu; in dma_4v_alloc_coherent() local
204 nid = dev->archdata.numa_node; in dma_4v_alloc_coherent()
212 iommu = dev->archdata.iommu; in dma_4v_alloc_coherent()
213 mask = dev->coherent_dma_mask; in dma_4v_alloc_coherent()
214 if (!iommu_use_atu(iommu, mask)) in dma_4v_alloc_coherent()
215 tbl = &iommu->tbl; in dma_4v_alloc_coherent()
217 tbl = &iommu->atu->tbl; in dma_4v_alloc_coherent()
220 (unsigned long)(-1), 0); in dma_4v_alloc_coherent()
225 *dma_addrp = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); in dma_4v_alloc_coherent()
268 list_for_each_entry(pdev, &bus_dev->devices, bus_list) { in dma_4v_iotsb_bind()
269 if (pdev->subordinate) { in dma_4v_iotsb_bind()
272 pdev->subordinate); in dma_4v_iotsb_bind()
274 bus = bus_dev->number; in dma_4v_iotsb_bind()
275 device = PCI_SLOT(pdev->devfn); in dma_4v_iotsb_bind()
276 fun = PCI_FUNC(pdev->devfn); in dma_4v_iotsb_bind()
317 npages -= num; in dma_4v_iommu_demap()
326 struct iommu *iommu; in dma_4v_free_coherent() local
334 iommu = dev->archdata.iommu; in dma_4v_free_coherent()
335 pbm = dev->archdata.host_controller; in dma_4v_free_coherent()
336 atu = iommu->atu; in dma_4v_free_coherent()
337 devhandle = pbm->devhandle; in dma_4v_free_coherent()
339 if (!iommu_use_atu(iommu, dvma)) { in dma_4v_free_coherent()
340 tbl = &iommu->tbl; in dma_4v_free_coherent()
341 iotsb_num = 0; /* we don't care for legacy iommu */ in dma_4v_free_coherent()
343 tbl = &atu->tbl; in dma_4v_free_coherent()
344 iotsb_num = atu->iotsb->iotsb_num; in dma_4v_free_coherent()
346 entry = ((dvma - tbl->table_map_base) >> IO_PAGE_SHIFT); in dma_4v_free_coherent()
359 struct iommu *iommu; in dma_4v_map_page() local
369 iommu = dev->archdata.iommu; in dma_4v_map_page()
370 atu = iommu->atu; in dma_4v_map_page()
376 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); in dma_4v_map_page()
379 mask = *dev->dma_mask; in dma_4v_map_page()
380 if (!iommu_use_atu(iommu, mask)) in dma_4v_map_page()
381 tbl = &iommu->tbl; in dma_4v_map_page()
383 tbl = &atu->tbl; in dma_4v_map_page()
386 (unsigned long)(-1), 0); in dma_4v_map_page()
391 bus_addr = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); in dma_4v_map_page()
433 struct iommu *iommu; in dma_4v_unmap_page() local
447 iommu = dev->archdata.iommu; in dma_4v_unmap_page()
448 pbm = dev->archdata.host_controller; in dma_4v_unmap_page()
449 atu = iommu->atu; in dma_4v_unmap_page()
450 devhandle = pbm->devhandle; in dma_4v_unmap_page()
452 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); in dma_4v_unmap_page()
457 iotsb_num = 0; /* we don't care for legacy iommu */ in dma_4v_unmap_page()
458 tbl = &iommu->tbl; in dma_4v_unmap_page()
460 iotsb_num = atu->iotsb->iotsb_num; in dma_4v_unmap_page()
461 tbl = &atu->tbl; in dma_4v_unmap_page()
463 entry = (bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT; in dma_4v_unmap_page()
478 struct iommu *iommu; in dma_4v_map_sg() local
487 iommu = dev->archdata.iommu; in dma_4v_map_sg()
488 if (nelems == 0 || !iommu) in dma_4v_map_sg()
490 atu = iommu->atu; in dma_4v_map_sg()
505 outs->dma_length = 0; in dma_4v_map_sg()
514 mask = *dev->dma_mask; in dma_4v_map_sg()
515 if (!iommu_use_atu(iommu, mask)) in dma_4v_map_sg()
516 tbl = &iommu->tbl; in dma_4v_map_sg()
518 tbl = &atu->tbl; in dma_4v_map_sg()
520 base_shift = tbl->table_map_base >> IO_PAGE_SHIFT; in dma_4v_map_sg()
525 slen = s->length; in dma_4v_map_sg()
531 /* Allocate iommu entries for that segment */ in dma_4v_map_sg()
535 &handle, (unsigned long)(-1), 0); in dma_4v_map_sg()
539 pr_err_ratelimited("iommu_alloc failed, iommu %p paddr %lx npages %lx\n", in dma_4v_map_sg()
547 dma_addr = tbl->table_map_base + (entry << IO_PAGE_SHIFT); in dma_4v_map_sg()
548 dma_addr |= (s->offset & ~IO_PAGE_MASK); in dma_4v_map_sg()
552 while (npages--) { in dma_4v_map_sg()
562 * - allocated dma_addr isn't contiguous to previous allocation in dma_4v_map_sg()
565 (outs->dma_length + s->length > max_seg_size) || in dma_4v_map_sg()
573 outs->dma_length += s->length; in dma_4v_map_sg()
579 outs->dma_address = dma_addr; in dma_4v_map_sg()
580 outs->dma_length = slen; in dma_4v_map_sg()
597 outs->dma_address = DMA_MAPPING_ERROR; in dma_4v_map_sg()
598 outs->dma_length = 0; in dma_4v_map_sg()
605 if (s->dma_length != 0) { in dma_4v_map_sg()
608 vaddr = s->dma_address & IO_PAGE_MASK; in dma_4v_map_sg()
609 npages = iommu_num_pages(s->dma_address, s->dma_length, in dma_4v_map_sg()
614 s->dma_address = DMA_MAPPING_ERROR; in dma_4v_map_sg()
615 s->dma_length = 0; in dma_4v_map_sg()
631 struct iommu *iommu; in dma_4v_unmap_sg() local
639 iommu = dev->archdata.iommu; in dma_4v_unmap_sg()
640 pbm = dev->archdata.host_controller; in dma_4v_unmap_sg()
641 atu = iommu->atu; in dma_4v_unmap_sg()
642 devhandle = pbm->devhandle; in dma_4v_unmap_sg()
647 while (nelems--) { in dma_4v_unmap_sg()
648 dma_addr_t dma_handle = sg->dma_address; in dma_4v_unmap_sg()
649 unsigned int len = sg->dma_length; in dma_4v_unmap_sg()
659 iotsb_num = 0; /* we don't care for legacy iommu */ in dma_4v_unmap_sg()
660 tbl = &iommu->tbl; in dma_4v_unmap_sg()
662 iotsb_num = atu->iotsb->iotsb_num; in dma_4v_unmap_sg()
663 tbl = &atu->tbl; in dma_4v_unmap_sg()
665 entry = ((dma_handle - tbl->table_map_base) >> shift); in dma_4v_unmap_sg()
678 struct iommu *iommu = dev->archdata.iommu; in dma_4v_supported() local
682 if (device_mask < iommu->dma_addr_mask) in dma_4v_supported()
702 dp = pbm->op->dev.of_node; in pci_sun4v_scan_bus()
703 prop = of_find_property(dp, "66mhz-capable", NULL); in pci_sun4v_scan_bus()
704 pbm->is_66mhz_capable = (prop != NULL); in pci_sun4v_scan_bus()
705 pbm->pci_bus = pci_scan_one_pbm(pbm, parent); in pci_sun4v_scan_bus()
711 struct iommu_map_table *iommu) in probe_existing_entries() argument
717 devhandle = pbm->devhandle; in probe_existing_entries()
718 for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) { in probe_existing_entries()
719 pool = &(iommu->pools[pool_nr]); in probe_existing_entries()
720 for (i = pool->start; i <= pool->end; i++) { in probe_existing_entries()
733 __set_bit(i, iommu->map); in probe_existing_entries()
743 struct atu *atu = pbm->iommu->atu; in pci_sun4v_atu_alloc_iotsb()
753 err = -ENOMEM; in pci_sun4v_atu_alloc_iotsb()
756 atu->iotsb = iotsb; in pci_sun4v_atu_alloc_iotsb()
759 table_size = (atu->size / IO_PAGE_SIZE) * 8; in pci_sun4v_atu_alloc_iotsb()
763 err = -ENOMEM; in pci_sun4v_atu_alloc_iotsb()
766 iotsb->table = table; in pci_sun4v_atu_alloc_iotsb()
767 iotsb->ra = __pa(table); in pci_sun4v_atu_alloc_iotsb()
768 iotsb->dvma_size = atu->size; in pci_sun4v_atu_alloc_iotsb()
769 iotsb->dvma_base = atu->base; in pci_sun4v_atu_alloc_iotsb()
770 iotsb->table_size = table_size; in pci_sun4v_atu_alloc_iotsb()
771 iotsb->page_size = IO_PAGE_SIZE; in pci_sun4v_atu_alloc_iotsb()
774 err = pci_sun4v_iotsb_conf(pbm->devhandle, in pci_sun4v_atu_alloc_iotsb()
775 iotsb->ra, in pci_sun4v_atu_alloc_iotsb()
776 iotsb->table_size, in pci_sun4v_atu_alloc_iotsb()
777 iotsb->page_size, in pci_sun4v_atu_alloc_iotsb()
778 iotsb->dvma_base, in pci_sun4v_atu_alloc_iotsb()
784 iotsb->iotsb_num = iotsb_num; in pci_sun4v_atu_alloc_iotsb()
786 err = dma_4v_iotsb_bind(pbm->devhandle, iotsb_num, pbm->pci_bus); in pci_sun4v_atu_alloc_iotsb()
804 struct atu *atu = pbm->iommu->atu; in pci_sun4v_atu_init()
812 ranges = of_get_property(pbm->op->dev.of_node, "iommu-address-ranges", in pci_sun4v_atu_init()
815 pr_err(PFX "No iommu-address-ranges\n"); in pci_sun4v_atu_init()
816 return -EINVAL; in pci_sun4v_atu_init()
819 page_size = of_get_property(pbm->op->dev.of_node, "iommu-pagesizes", in pci_sun4v_atu_init()
822 pr_err(PFX "No iommu-pagesizes\n"); in pci_sun4v_atu_init()
823 return -EINVAL; in pci_sun4v_atu_init()
826 /* There are 4 iommu-address-ranges supported. Each range is pair of in pci_sun4v_atu_init()
836 atu->ranges = (struct atu_ranges *)ranges; in pci_sun4v_atu_init()
837 atu->base = atu->ranges[3].base; in pci_sun4v_atu_init()
838 atu->size = ATU_64_SPACE_SIZE; in pci_sun4v_atu_init()
847 /* Create ATU iommu map. in pci_sun4v_atu_init()
850 dma_mask = (roundup_pow_of_two(atu->size) - 1UL); in pci_sun4v_atu_init()
851 num_iotte = atu->size / IO_PAGE_SIZE; in pci_sun4v_atu_init()
853 atu->tbl.table_map_base = atu->base; in pci_sun4v_atu_init()
854 atu->dma_addr_mask = dma_mask; in pci_sun4v_atu_init()
855 atu->tbl.map = kzalloc(map_size, GFP_KERNEL); in pci_sun4v_atu_init()
856 if (!atu->tbl.map) in pci_sun4v_atu_init()
857 return -ENOMEM; in pci_sun4v_atu_init()
859 iommu_tbl_pool_init(&atu->tbl, num_iotte, IO_PAGE_SHIFT, in pci_sun4v_atu_init()
870 struct iommu *iommu = pbm->iommu; in pci_sun4v_iommu_init() local
875 vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", NULL); in pci_sun4v_iommu_init()
880 printk(KERN_ERR PFX "Strange virtual-dma[%08x:%08x].\n", in pci_sun4v_iommu_init()
882 return -EINVAL; in pci_sun4v_iommu_init()
885 dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL); in pci_sun4v_iommu_init()
890 /* Setup initial software IOMMU state. */ in pci_sun4v_iommu_init()
891 spin_lock_init(&iommu->lock); in pci_sun4v_iommu_init()
892 iommu->ctx_lowest_free = 1; in pci_sun4v_iommu_init()
893 iommu->tbl.table_map_base = dma_offset; in pci_sun4v_iommu_init()
894 iommu->dma_addr_mask = dma_mask; in pci_sun4v_iommu_init()
896 /* Allocate and initialize the free area map. */ in pci_sun4v_iommu_init()
899 iommu->tbl.map = kzalloc(sz, GFP_KERNEL); in pci_sun4v_iommu_init()
900 if (!iommu->tbl.map) { in pci_sun4v_iommu_init()
901 printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); in pci_sun4v_iommu_init()
902 return -ENOMEM; in pci_sun4v_iommu_init()
904 iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, in pci_sun4v_iommu_init()
908 sz = probe_existing_entries(pbm, &iommu->tbl); in pci_sun4v_iommu_init()
911 pbm->name, sz); in pci_sun4v_iommu_init()
945 * For MSI-X bits 31:0 are the data from the MSI packet.
947 * bits 39:32 is the bus/device/fn of the msg target-id
950 * For INTx the low order 2-bits are:
951 * 00 - INTA
952 * 01 - INTB
953 * 10 - INTC
954 * 11 - INTD
966 err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, head); in pci_sun4v_get_head()
968 return -ENXIO; in pci_sun4v_get_head()
970 limit = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); in pci_sun4v_get_head()
972 return -EFBIG; in pci_sun4v_get_head()
985 ep = (pbm->msi_queues + ((msiqid - pbm->msiq_first) * in pci_sun4v_dequeue_msi()
986 (pbm->msiq_ent_count * in pci_sun4v_dequeue_msi()
990 if ((ep->version_type & MSIQ_TYPE_MASK) == 0) in pci_sun4v_dequeue_msi()
993 type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT; in pci_sun4v_dequeue_msi()
996 return -EINVAL; in pci_sun4v_dequeue_msi()
998 *msi = ep->msi_data; in pci_sun4v_dequeue_msi()
1000 err = pci_sun4v_msi_setstate(pbm->devhandle, in pci_sun4v_dequeue_msi()
1001 ep->msi_data /* msi_num */, in pci_sun4v_dequeue_msi()
1004 return -ENXIO; in pci_sun4v_dequeue_msi()
1007 ep->version_type &= ~MSIQ_TYPE_MASK; in pci_sun4v_dequeue_msi()
1011 (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry))) in pci_sun4v_dequeue_msi()
1022 err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head); in pci_sun4v_set_head()
1024 return -EINVAL; in pci_sun4v_set_head()
1032 if (pci_sun4v_msi_setmsiq(pbm->devhandle, msi, msiqid, in pci_sun4v_msi_setup()
1035 return -ENXIO; in pci_sun4v_msi_setup()
1036 if (pci_sun4v_msi_setstate(pbm->devhandle, msi, HV_MSISTATE_IDLE)) in pci_sun4v_msi_setup()
1037 return -ENXIO; in pci_sun4v_msi_setup()
1038 if (pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_VALID)) in pci_sun4v_msi_setup()
1039 return -ENXIO; in pci_sun4v_msi_setup()
1047 err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi, &msiqid); in pci_sun4v_msi_teardown()
1049 return -ENXIO; in pci_sun4v_msi_teardown()
1051 pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_INVALID); in pci_sun4v_msi_teardown()
1061 q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); in pci_sun4v_msiq_alloc()
1062 alloc_size = (pbm->msiq_num * q_size); in pci_sun4v_msiq_alloc()
1068 return -ENOMEM; in pci_sun4v_msiq_alloc()
1071 pbm->msi_queues = (void *) pages; in pci_sun4v_msiq_alloc()
1073 for (i = 0; i < pbm->msiq_num; i++) { in pci_sun4v_msiq_alloc()
1077 err = pci_sun4v_msiq_conf(pbm->devhandle, in pci_sun4v_msiq_alloc()
1078 pbm->msiq_first + i, in pci_sun4v_msiq_alloc()
1079 base, pbm->msiq_ent_count); in pci_sun4v_msiq_alloc()
1086 err = pci_sun4v_msiq_info(pbm->devhandle, in pci_sun4v_msiq_alloc()
1087 pbm->msiq_first + i, in pci_sun4v_msiq_alloc()
1094 if (ret1 != base || ret2 != pbm->msiq_ent_count) { in pci_sun4v_msiq_alloc()
1097 base, pbm->msiq_ent_count, in pci_sun4v_msiq_alloc()
1107 return -EINVAL; in pci_sun4v_msiq_alloc()
1115 for (i = 0; i < pbm->msiq_num; i++) { in pci_sun4v_msiq_free()
1116 unsigned long msiqid = pbm->msiq_first + i; in pci_sun4v_msiq_free()
1118 (void) pci_sun4v_msiq_conf(pbm->devhandle, msiqid, 0UL, 0); in pci_sun4v_msiq_free()
1121 q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); in pci_sun4v_msiq_free()
1122 alloc_size = (pbm->msiq_num * q_size); in pci_sun4v_msiq_free()
1125 pages = (unsigned long) pbm->msi_queues; in pci_sun4v_msiq_free()
1129 pbm->msi_queues = NULL; in pci_sun4v_msiq_free()
1136 unsigned int irq = sun4v_build_irq(pbm->devhandle, devino); in pci_sun4v_msiq_build_irq()
1139 return -ENOMEM; in pci_sun4v_msiq_build_irq()
1141 if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID)) in pci_sun4v_msiq_build_irq()
1142 return -EINVAL; in pci_sun4v_msiq_build_irq()
1143 if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) in pci_sun4v_msiq_build_irq()
1144 return -EINVAL; in pci_sun4v_msiq_build_irq()
1173 struct device_node *dp = op->dev.of_node; in pci_sun4v_pbm_init()
1176 pbm->numa_node = of_node_to_nid(dp); in pci_sun4v_pbm_init()
1178 pbm->pci_ops = &sun4v_pci_ops; in pci_sun4v_pbm_init()
1179 pbm->config_space_reg_bits = 12; in pci_sun4v_pbm_init()
1181 pbm->index = pci_num_pbms++; in pci_sun4v_pbm_init()
1183 pbm->op = op; in pci_sun4v_pbm_init()
1185 pbm->devhandle = devhandle; in pci_sun4v_pbm_init()
1187 pbm->name = dp->full_name; in pci_sun4v_pbm_init()
1189 printk("%s: SUN4V PCI Bus Module\n", pbm->name); in pci_sun4v_pbm_init()
1190 printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); in pci_sun4v_pbm_init()
1202 pci_sun4v_scan_bus(pbm, &op->dev); in pci_sun4v_pbm_init()
1205 * we can still continue using legacy iommu. in pci_sun4v_pbm_init()
1207 if (pbm->iommu->atu) { in pci_sun4v_pbm_init()
1210 kfree(pbm->iommu->atu); in pci_sun4v_pbm_init()
1211 pbm->iommu->atu = NULL; in pci_sun4v_pbm_init()
1216 pbm->next = pci_pbm_root; in pci_sun4v_pbm_init()
1228 struct iommu *iommu; in pci_sun4v_probe() local
1231 int i, err = -ENODEV; in pci_sun4v_probe()
1234 dp = op->dev.of_node; in pci_sun4v_probe()
1269 err = -ENODEV; in pci_sun4v_probe()
1274 devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff; in pci_sun4v_probe()
1276 err = -ENOMEM; in pci_sun4v_probe()
1295 iommu = kzalloc(sizeof(struct iommu), GFP_KERNEL); in pci_sun4v_probe()
1296 if (!iommu) { in pci_sun4v_probe()
1297 printk(KERN_ERR PFX "Could not allocate pbm iommu\n"); in pci_sun4v_probe()
1301 pbm->iommu = iommu; in pci_sun4v_probe()
1302 iommu->atu = NULL; in pci_sun4v_probe()
1308 iommu->atu = atu; in pci_sun4v_probe()
1315 dev_set_drvdata(&op->dev, pbm); in pci_sun4v_probe()
1320 kfree(iommu->atu); in pci_sun4v_probe()
1321 kfree(pbm->iommu); in pci_sun4v_probe()
1333 .compatible = "SUNW,sun4v-pci",