Lines Matching +full:data +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
3 * CPU-agnostic ARM page table allocator.
10 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt
14 #include <linux/io-pgtable.h>
19 #include <linux/dma-mapping.h>
23 #include "io-pgtable-arm.h"
41 (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
45 (sizeof(arm_lpae_iopte) << (d)->bits_per_level)
47 (sizeof(arm_lpae_iopte) << (d)->pgd_bits)
57 ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
61 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
63 /* Calculate the block/page mapping size at level l for pagetable in d. */
93 /* Stage-1 PTE */
99 /* Stage-2 PTE */
159 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) in iopte_leaf()
166 struct arm_lpae_io_pgtable *data) in paddr_to_iopte() argument
171 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; in paddr_to_iopte()
175 struct arm_lpae_io_pgtable *data) in iopte_to_paddr() argument
179 if (ARM_LPAE_GRANULE(data) < SZ_64K) in iopte_to_paddr()
182 /* Rotate the packed high-order bits back to the top */ in iopte_to_paddr()
183 return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); in iopte_to_paddr()
193 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, in __arm_lpae_alloc_pages() argument
196 struct device *dev = cfg->iommu_dev; in __arm_lpae_alloc_pages()
197 int order = get_order(size); in __arm_lpae_alloc_pages()
209 if (!cfg->coherent_walk) { in __arm_lpae_alloc_pages()
210 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); in __arm_lpae_alloc_pages()
226 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); in __arm_lpae_alloc_pages()
232 static void __arm_lpae_free_pages(void *pages, size_t size, in __arm_lpae_free_pages() argument
235 if (!cfg->coherent_walk) in __arm_lpae_free_pages()
236 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), in __arm_lpae_free_pages()
237 size, DMA_TO_DEVICE); in __arm_lpae_free_pages()
238 free_pages((unsigned long)pages, get_order(size)); in __arm_lpae_free_pages()
244 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), in __arm_lpae_sync_pte()
253 if (!cfg->coherent_walk) in __arm_lpae_clear_pte()
257 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
259 unsigned long iova, size_t size, size_t pgcount,
262 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, in __arm_lpae_init_pte() argument
267 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_lpae_init_pte()
268 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); in __arm_lpae_init_pte()
271 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) in __arm_lpae_init_pte()
277 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); in __arm_lpae_init_pte()
279 if (!cfg->coherent_walk) in __arm_lpae_init_pte()
283 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, in arm_lpae_init_pte() argument
291 if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) { in arm_lpae_init_pte()
294 return -EEXIST; in arm_lpae_init_pte()
301 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); in arm_lpae_init_pte()
303 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); in arm_lpae_init_pte()
304 if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1, in arm_lpae_init_pte()
307 return -EINVAL; in arm_lpae_init_pte()
311 __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep); in arm_lpae_init_pte()
323 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_install_table()
335 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) in arm_lpae_install_table()
346 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, in __arm_lpae_map() argument
347 phys_addr_t paddr, size_t size, size_t pgcount, in __arm_lpae_map() argument
352 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); in __arm_lpae_map()
353 size_t tblsz = ARM_LPAE_GRANULE(data); in __arm_lpae_map()
354 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_lpae_map()
358 map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); in __arm_lpae_map()
362 if (size == block_size) { in __arm_lpae_map()
363 max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start; in __arm_lpae_map()
365 ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); in __arm_lpae_map()
367 *mapped += num_entries * size; in __arm_lpae_map()
373 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) in __arm_lpae_map()
374 return -EINVAL; in __arm_lpae_map()
381 return -ENOMEM; in __arm_lpae_map()
386 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { in __arm_lpae_map()
390 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { in __arm_lpae_map()
391 cptep = iopte_deref(pte, data); in __arm_lpae_map()
395 return -EEXIST; in __arm_lpae_map()
399 return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1, in __arm_lpae_map()
403 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, in arm_lpae_prot_to_pte() argument
408 if (data->iop.fmt == APPLE_DART) { in arm_lpae_prot_to_pte()
417 if (data->iop.fmt == ARM_64_LPAE_S1 || in arm_lpae_prot_to_pte()
418 data->iop.fmt == ARM_32_LPAE_S1) { in arm_lpae_prot_to_pte()
434 * having stage-1-like attributes but stage-2-like permissions. in arm_lpae_prot_to_pte()
436 if (data->iop.fmt == ARM_64_LPAE_S2 || in arm_lpae_prot_to_pte()
437 data->iop.fmt == ARM_32_LPAE_S2) { in arm_lpae_prot_to_pte()
459 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE) in arm_lpae_prot_to_pte()
467 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_prot_to_pte()
470 if (data->iop.fmt != ARM_MALI_LPAE) in arm_lpae_prot_to_pte()
480 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_map_pages() local
481 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_map_pages()
482 arm_lpae_iopte *ptep = data->pgd; in arm_lpae_map_pages()
483 int ret, lvl = data->start_level; in arm_lpae_map_pages()
485 long iaext = (s64)iova >> cfg->ias; in arm_lpae_map_pages()
487 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) in arm_lpae_map_pages()
488 return -EINVAL; in arm_lpae_map_pages()
490 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_map_pages()
492 if (WARN_ON(iaext || paddr >> cfg->oas)) in arm_lpae_map_pages()
493 return -ERANGE; in arm_lpae_map_pages()
499 prot = arm_lpae_prot_to_pte(data, iommu_prot); in arm_lpae_map_pages()
500 ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl, in arm_lpae_map_pages()
512 phys_addr_t paddr, size_t size, int iommu_prot, gfp_t gfp) in arm_lpae_map() argument
514 return arm_lpae_map_pages(ops, iova, paddr, size, 1, iommu_prot, gfp, in arm_lpae_map()
518 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, in __arm_lpae_free_pgtable() argument
524 if (lvl == data->start_level) in __arm_lpae_free_pgtable()
525 table_size = ARM_LPAE_PGD_SIZE(data); in __arm_lpae_free_pgtable()
527 table_size = ARM_LPAE_GRANULE(data); in __arm_lpae_free_pgtable()
532 if (lvl == ARM_LPAE_MAX_LEVELS - 1) in __arm_lpae_free_pgtable()
540 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) in __arm_lpae_free_pgtable()
543 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); in __arm_lpae_free_pgtable()
546 __arm_lpae_free_pages(start, table_size, &data->iop.cfg); in __arm_lpae_free_pgtable()
551 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); in arm_lpae_free_pgtable() local
553 __arm_lpae_free_pgtable(data, data->start_level, data->pgd); in arm_lpae_free_pgtable()
554 kfree(data); in arm_lpae_free_pgtable()
557 static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, in arm_lpae_split_blk_unmap() argument
559 unsigned long iova, size_t size, in arm_lpae_split_blk_unmap() argument
563 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_split_blk_unmap()
566 size_t tablesz = ARM_LPAE_GRANULE(data); in arm_lpae_split_blk_unmap()
567 size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); in arm_lpae_split_blk_unmap()
568 int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data); in arm_lpae_split_blk_unmap()
569 int i, unmap_idx_start = -1, num_entries = 0, max_entries; in arm_lpae_split_blk_unmap()
578 if (size == split_sz) { in arm_lpae_split_blk_unmap()
579 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); in arm_lpae_split_blk_unmap()
580 max_entries = ptes_per_table - unmap_idx_start; in arm_lpae_split_blk_unmap()
584 blk_paddr = iopte_to_paddr(blk_pte, data); in arm_lpae_split_blk_unmap()
592 __arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]); in arm_lpae_split_blk_unmap()
606 tablep = iopte_deref(pte, data); in arm_lpae_split_blk_unmap()
609 io_pgtable_tlb_add_page(&data->iop, gather, iova + i * size, size); in arm_lpae_split_blk_unmap()
611 return num_entries * size; in arm_lpae_split_blk_unmap()
614 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep); in arm_lpae_split_blk_unmap()
617 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, in __arm_lpae_unmap() argument
619 unsigned long iova, size_t size, size_t pgcount, in __arm_lpae_unmap() argument
623 struct io_pgtable *iop = &data->iop; in __arm_lpae_unmap()
630 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); in __arm_lpae_unmap()
636 /* If the size matches this level, we're in the right place */ in __arm_lpae_unmap()
637 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { in __arm_lpae_unmap()
638 max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start; in __arm_lpae_unmap()
646 __arm_lpae_clear_pte(ptep, &iop->cfg); in __arm_lpae_unmap()
648 if (!iopte_leaf(pte, lvl, iop->fmt)) { in __arm_lpae_unmap()
650 io_pgtable_tlb_flush_walk(iop, iova + i * size, size, in __arm_lpae_unmap()
651 ARM_LPAE_GRANULE(data)); in __arm_lpae_unmap()
652 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); in __arm_lpae_unmap()
654 io_pgtable_tlb_add_page(iop, gather, iova + i * size, size); in __arm_lpae_unmap()
661 return i * size; in __arm_lpae_unmap()
662 } else if (iopte_leaf(pte, lvl, iop->fmt)) { in __arm_lpae_unmap()
667 return arm_lpae_split_blk_unmap(data, gather, iova, size, pte, in __arm_lpae_unmap()
672 ptep = iopte_deref(pte, data); in __arm_lpae_unmap()
673 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep); in __arm_lpae_unmap()
680 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_unmap_pages() local
681 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_unmap_pages()
682 arm_lpae_iopte *ptep = data->pgd; in arm_lpae_unmap_pages()
683 long iaext = (s64)iova >> cfg->ias; in arm_lpae_unmap_pages()
685 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) in arm_lpae_unmap_pages()
688 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_unmap_pages()
693 return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount, in arm_lpae_unmap_pages()
694 data->start_level, ptep); in arm_lpae_unmap_pages()
698 size_t size, struct iommu_iotlb_gather *gather) in arm_lpae_unmap() argument
700 return arm_lpae_unmap_pages(ops, iova, size, 1, gather); in arm_lpae_unmap()
706 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_iova_to_phys() local
707 arm_lpae_iopte pte, *ptep = data->pgd; in arm_lpae_iova_to_phys()
708 int lvl = data->start_level; in arm_lpae_iova_to_phys()
716 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); in arm_lpae_iova_to_phys()
724 if (iopte_leaf(pte, lvl, data->iop.fmt)) in arm_lpae_iova_to_phys()
728 ptep = iopte_deref(pte, data); in arm_lpae_iova_to_phys()
735 iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); in arm_lpae_iova_to_phys()
736 return iopte_to_paddr(pte, data) | iova; in arm_lpae_iova_to_phys()
747 * the CPU page size if possible, otherwise prefer smaller sizes. in arm_lpae_restrict_pgsizes()
751 if (cfg->pgsize_bitmap & PAGE_SIZE) in arm_lpae_restrict_pgsizes()
753 else if (cfg->pgsize_bitmap & ~PAGE_MASK) in arm_lpae_restrict_pgsizes()
754 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); in arm_lpae_restrict_pgsizes()
755 else if (cfg->pgsize_bitmap & PAGE_MASK) in arm_lpae_restrict_pgsizes()
756 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); in arm_lpae_restrict_pgsizes()
770 if (cfg->oas > 48) in arm_lpae_restrict_pgsizes()
777 cfg->pgsize_bitmap &= page_sizes; in arm_lpae_restrict_pgsizes()
778 cfg->ias = min(cfg->ias, max_addr_bits); in arm_lpae_restrict_pgsizes()
779 cfg->oas = min(cfg->oas, max_addr_bits); in arm_lpae_restrict_pgsizes()
785 struct arm_lpae_io_pgtable *data; in arm_lpae_alloc_pgtable() local
790 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) in arm_lpae_alloc_pgtable()
793 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
796 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
799 data = kmalloc(sizeof(*data), GFP_KERNEL); in arm_lpae_alloc_pgtable()
800 if (!data) in arm_lpae_alloc_pgtable()
803 pg_shift = __ffs(cfg->pgsize_bitmap); in arm_lpae_alloc_pgtable()
804 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte)); in arm_lpae_alloc_pgtable()
806 va_bits = cfg->ias - pg_shift; in arm_lpae_alloc_pgtable()
807 levels = DIV_ROUND_UP(va_bits, data->bits_per_level); in arm_lpae_alloc_pgtable()
808 data->start_level = ARM_LPAE_MAX_LEVELS - levels; in arm_lpae_alloc_pgtable()
810 /* Calculate the actual size of our pgd (without concatenation) */ in arm_lpae_alloc_pgtable()
811 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); in arm_lpae_alloc_pgtable()
813 data->iop.ops = (struct io_pgtable_ops) { in arm_lpae_alloc_pgtable()
821 return data; in arm_lpae_alloc_pgtable()
828 struct arm_lpae_io_pgtable *data; in arm_64_lpae_alloc_pgtable_s1() local
829 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; in arm_64_lpae_alloc_pgtable_s1()
832 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | in arm_64_lpae_alloc_pgtable_s1()
837 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s1()
838 if (!data) in arm_64_lpae_alloc_pgtable_s1()
842 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s1()
843 tcr->sh = ARM_LPAE_TCR_SH_IS; in arm_64_lpae_alloc_pgtable_s1()
844 tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s1()
845 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s1()
846 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA) in arm_64_lpae_alloc_pgtable_s1()
849 tcr->sh = ARM_LPAE_TCR_SH_OS; in arm_64_lpae_alloc_pgtable_s1()
850 tcr->irgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s1()
851 if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) in arm_64_lpae_alloc_pgtable_s1()
852 tcr->orgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s1()
854 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s1()
857 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; in arm_64_lpae_alloc_pgtable_s1()
858 switch (ARM_LPAE_GRANULE(data)) { in arm_64_lpae_alloc_pgtable_s1()
860 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K; in arm_64_lpae_alloc_pgtable_s1()
863 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K; in arm_64_lpae_alloc_pgtable_s1()
866 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K; in arm_64_lpae_alloc_pgtable_s1()
870 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s1()
872 tcr->ips = ARM_LPAE_TCR_PS_32_BIT; in arm_64_lpae_alloc_pgtable_s1()
875 tcr->ips = ARM_LPAE_TCR_PS_36_BIT; in arm_64_lpae_alloc_pgtable_s1()
878 tcr->ips = ARM_LPAE_TCR_PS_40_BIT; in arm_64_lpae_alloc_pgtable_s1()
881 tcr->ips = ARM_LPAE_TCR_PS_42_BIT; in arm_64_lpae_alloc_pgtable_s1()
884 tcr->ips = ARM_LPAE_TCR_PS_44_BIT; in arm_64_lpae_alloc_pgtable_s1()
887 tcr->ips = ARM_LPAE_TCR_PS_48_BIT; in arm_64_lpae_alloc_pgtable_s1()
890 tcr->ips = ARM_LPAE_TCR_PS_52_BIT; in arm_64_lpae_alloc_pgtable_s1()
896 tcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s1()
908 cfg->arm_lpae_s1_cfg.mair = reg; in arm_64_lpae_alloc_pgtable_s1()
911 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), in arm_64_lpae_alloc_pgtable_s1()
913 if (!data->pgd) in arm_64_lpae_alloc_pgtable_s1()
920 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s1()
921 return &data->iop; in arm_64_lpae_alloc_pgtable_s1()
924 kfree(data); in arm_64_lpae_alloc_pgtable_s1()
932 struct arm_lpae_io_pgtable *data; in arm_64_lpae_alloc_pgtable_s2() local
933 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; in arm_64_lpae_alloc_pgtable_s2()
936 if (cfg->quirks) in arm_64_lpae_alloc_pgtable_s2()
939 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s2()
940 if (!data) in arm_64_lpae_alloc_pgtable_s2()
945 * the depth of the stage-2 walk. in arm_64_lpae_alloc_pgtable_s2()
947 if (data->start_level == 0) { in arm_64_lpae_alloc_pgtable_s2()
950 pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte); in arm_64_lpae_alloc_pgtable_s2()
952 data->pgd_bits += data->bits_per_level; in arm_64_lpae_alloc_pgtable_s2()
953 data->start_level++; in arm_64_lpae_alloc_pgtable_s2()
958 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s2()
959 vtcr->sh = ARM_LPAE_TCR_SH_IS; in arm_64_lpae_alloc_pgtable_s2()
960 vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s2()
961 vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA; in arm_64_lpae_alloc_pgtable_s2()
963 vtcr->sh = ARM_LPAE_TCR_SH_OS; in arm_64_lpae_alloc_pgtable_s2()
964 vtcr->irgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s2()
965 vtcr->orgn = ARM_LPAE_TCR_RGN_NC; in arm_64_lpae_alloc_pgtable_s2()
968 sl = data->start_level; in arm_64_lpae_alloc_pgtable_s2()
970 switch (ARM_LPAE_GRANULE(data)) { in arm_64_lpae_alloc_pgtable_s2()
972 vtcr->tg = ARM_LPAE_TCR_TG0_4K; in arm_64_lpae_alloc_pgtable_s2()
973 sl++; /* SL0 format is different for 4K granule size */ in arm_64_lpae_alloc_pgtable_s2()
976 vtcr->tg = ARM_LPAE_TCR_TG0_16K; in arm_64_lpae_alloc_pgtable_s2()
979 vtcr->tg = ARM_LPAE_TCR_TG0_64K; in arm_64_lpae_alloc_pgtable_s2()
983 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s2()
985 vtcr->ps = ARM_LPAE_TCR_PS_32_BIT; in arm_64_lpae_alloc_pgtable_s2()
988 vtcr->ps = ARM_LPAE_TCR_PS_36_BIT; in arm_64_lpae_alloc_pgtable_s2()
991 vtcr->ps = ARM_LPAE_TCR_PS_40_BIT; in arm_64_lpae_alloc_pgtable_s2()
994 vtcr->ps = ARM_LPAE_TCR_PS_42_BIT; in arm_64_lpae_alloc_pgtable_s2()
997 vtcr->ps = ARM_LPAE_TCR_PS_44_BIT; in arm_64_lpae_alloc_pgtable_s2()
1000 vtcr->ps = ARM_LPAE_TCR_PS_48_BIT; in arm_64_lpae_alloc_pgtable_s2()
1003 vtcr->ps = ARM_LPAE_TCR_PS_52_BIT; in arm_64_lpae_alloc_pgtable_s2()
1009 vtcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s2()
1010 vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK; in arm_64_lpae_alloc_pgtable_s2()
1013 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), in arm_64_lpae_alloc_pgtable_s2()
1015 if (!data->pgd) in arm_64_lpae_alloc_pgtable_s2()
1022 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s2()
1023 return &data->iop; in arm_64_lpae_alloc_pgtable_s2()
1026 kfree(data); in arm_64_lpae_alloc_pgtable_s2()
1033 if (cfg->ias > 32 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s1()
1036 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s1()
1043 if (cfg->ias > 40 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s2()
1046 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s2()
1053 struct arm_lpae_io_pgtable *data; in arm_mali_lpae_alloc_pgtable() local
1056 if (cfg->quirks) in arm_mali_lpae_alloc_pgtable()
1059 if (cfg->ias > 48 || cfg->oas > 40) in arm_mali_lpae_alloc_pgtable()
1062 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_mali_lpae_alloc_pgtable()
1064 data = arm_lpae_alloc_pgtable(cfg); in arm_mali_lpae_alloc_pgtable()
1065 if (!data) in arm_mali_lpae_alloc_pgtable()
1068 /* Mali seems to need a full 4-level table regardless of IAS */ in arm_mali_lpae_alloc_pgtable()
1069 if (data->start_level > 0) { in arm_mali_lpae_alloc_pgtable()
1070 data->start_level = 0; in arm_mali_lpae_alloc_pgtable()
1071 data->pgd_bits = 0; in arm_mali_lpae_alloc_pgtable()
1074 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the in arm_mali_lpae_alloc_pgtable()
1075 * best we can do is mimic the out-of-tree driver and hope that the in arm_mali_lpae_alloc_pgtable()
1076 * "implementation-defined caching policy" is good enough. Similarly, in arm_mali_lpae_alloc_pgtable()
1080 cfg->arm_mali_lpae_cfg.memattr = in arm_mali_lpae_alloc_pgtable()
1088 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, in arm_mali_lpae_alloc_pgtable()
1090 if (!data->pgd) in arm_mali_lpae_alloc_pgtable()
1096 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | in arm_mali_lpae_alloc_pgtable()
1099 if (cfg->coherent_walk) in arm_mali_lpae_alloc_pgtable()
1100 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER; in arm_mali_lpae_alloc_pgtable()
1102 return &data->iop; in arm_mali_lpae_alloc_pgtable()
1105 kfree(data); in arm_mali_lpae_alloc_pgtable()
1112 struct arm_lpae_io_pgtable *data; in apple_dart_alloc_pgtable() local
1115 if (cfg->oas > 36) in apple_dart_alloc_pgtable()
1118 data = arm_lpae_alloc_pgtable(cfg); in apple_dart_alloc_pgtable()
1119 if (!data) in apple_dart_alloc_pgtable()
1129 if (data->start_level < 1) in apple_dart_alloc_pgtable()
1131 if (data->start_level == 1 && data->pgd_bits > 2) in apple_dart_alloc_pgtable()
1133 if (data->start_level > 1) in apple_dart_alloc_pgtable()
1134 data->pgd_bits = 0; in apple_dart_alloc_pgtable()
1135 data->start_level = 2; in apple_dart_alloc_pgtable()
1136 cfg->apple_dart_cfg.n_ttbrs = 1 << data->pgd_bits; in apple_dart_alloc_pgtable()
1137 data->pgd_bits += data->bits_per_level; in apple_dart_alloc_pgtable()
1139 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, in apple_dart_alloc_pgtable()
1141 if (!data->pgd) in apple_dart_alloc_pgtable()
1144 for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) in apple_dart_alloc_pgtable()
1145 cfg->apple_dart_cfg.ttbr[i] = in apple_dart_alloc_pgtable()
1146 virt_to_phys(data->pgd + i * ARM_LPAE_GRANULE(data)); in apple_dart_alloc_pgtable()
1148 return &data->iop; in apple_dart_alloc_pgtable()
1151 kfree(data); in apple_dart_alloc_pgtable()
1194 static void __init dummy_tlb_flush(unsigned long iova, size_t size, in dummy_tlb_flush() argument
1198 WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); in dummy_tlb_flush()
1216 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); in arm_lpae_dump_ops() local
1217 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_dump_ops()
1219 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", in arm_lpae_dump_ops()
1220 cfg->pgsize_bitmap, cfg->ias); in arm_lpae_dump_ops()
1221 pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n", in arm_lpae_dump_ops()
1222 ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data), in arm_lpae_dump_ops()
1223 ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd); in arm_lpae_dump_ops()
1230 -EFAULT; \
1242 size_t size; in arm_lpae_run_tests() local
1252 return -ENOMEM; in arm_lpae_run_tests()
1259 if (ops->iova_to_phys(ops, 42)) in arm_lpae_run_tests()
1262 if (ops->iova_to_phys(ops, SZ_1G + 42)) in arm_lpae_run_tests()
1265 if (ops->iova_to_phys(ops, SZ_2G + 42)) in arm_lpae_run_tests()
1272 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1273 size = 1UL << j; in arm_lpae_run_tests()
1275 if (ops->map(ops, iova, iova, size, IOMMU_READ | in arm_lpae_run_tests()
1282 if (!ops->map(ops, iova, iova + size, size, in arm_lpae_run_tests()
1286 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) in arm_lpae_run_tests()
1293 size = 1UL << __ffs(cfg->pgsize_bitmap); in arm_lpae_run_tests()
1294 if (ops->unmap(ops, SZ_1G + size, size, NULL) != size) in arm_lpae_run_tests()
1298 if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ, GFP_KERNEL)) in arm_lpae_run_tests()
1301 if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) in arm_lpae_run_tests()
1306 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1307 size = 1UL << j; in arm_lpae_run_tests()
1309 if (ops->unmap(ops, iova, size, NULL) != size) in arm_lpae_run_tests()
1312 if (ops->iova_to_phys(ops, iova + 42)) in arm_lpae_run_tests()
1316 if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL)) in arm_lpae_run_tests()
1319 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) in arm_lpae_run_tests()
1365 return fail ? -EFAULT : 0; in arm_lpae_do_selftests()