1 // SPDX-License-Identifier: GPL-2.0-only
3  * CPU-agnostic ARM page table allocator.
5  * ARMv7 Short-descriptor format, supporting
6  * - Basic memory attributes
7  * - Simplified access permissions (AP[2:1] model)
8  * - Backwards-compatible TEX remap
9  * - Large pages/supersections (if indicated by the caller)
12  * - Legacy access permissions (AP[2:0] model)
15  * - PXN
16  * - Domains
18  * Copyright (C) 2014-2015 ARM Limited
19  * Copyright (c) 2014-2015 MediaTek Inc.
22 #define pr_fmt(fmt)	"arm-v7s io-pgtable: " fmt
25 #include <linux/dma-mapping.h>
27 #include <linux/io-pgtable.h>
51 #define _ARM_V7S_LVL_BITS(lvl, cfg)	((lvl) == 1 ? ((cfg)->ias - 20) : 8)
62 #define _ARM_V7S_IDX_MASK(lvl, cfg)	(ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1)
88 #define ARM_V7S_ATTR_XN(lvl)		BIT(4 * (2 - (lvl)))
100  * fields relative to that 8-bit block, plus a total shift relative to the PTE.
102 #define ARM_V7S_ATTR_SHIFT(lvl)		(16 - (lvl) * 6)
182 		(cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT);  in arm_v7s_is_mtk_enabled()
229 				  struct arm_v7s_io_pgtable *data)  in iopte_deref()  argument
231 	return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg));  in iopte_deref()
235 				   struct arm_v7s_io_pgtable *data)  in __arm_v7s_alloc_table()  argument
237 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_alloc_table()
238 	struct device *dev = cfg->iommu_dev;  in __arm_v7s_alloc_table()
241 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_alloc_table()  local
246 			__GFP_ZERO | ARM_V7S_TABLE_GFP_DMA, get_order(size));  in __arm_v7s_alloc_table()
248 		table = kmem_cache_zalloc(data->l2_tables, gfp);  in __arm_v7s_alloc_table()
255 	if (table && !cfg->coherent_walk) {  in __arm_v7s_alloc_table()
256 		dma = dma_map_single(dev, table, size, DMA_TO_DEVICE);  in __arm_v7s_alloc_table()
273 	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);  in __arm_v7s_alloc_table()
276 		free_pages((unsigned long)table, get_order(size));  in __arm_v7s_alloc_table()
278 		kmem_cache_free(data->l2_tables, table);  in __arm_v7s_alloc_table()
283 				 struct arm_v7s_io_pgtable *data)  in __arm_v7s_free_table()  argument
285 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_free_table()
286 	struct device *dev = cfg->iommu_dev;  in __arm_v7s_free_table()
287 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_free_table()  local
289 	if (!cfg->coherent_walk)  in __arm_v7s_free_table()
290 		dma_unmap_single(dev, __arm_v7s_dma_addr(table), size,  in __arm_v7s_free_table()
293 		free_pages((unsigned long)table, get_order(size));  in __arm_v7s_free_table()
295 		kmem_cache_free(data->l2_tables, table);  in __arm_v7s_free_table()
301 	if (cfg->coherent_walk)  in __arm_v7s_pte_sync()
304 	dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep),  in __arm_v7s_pte_sync()
321 	bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS);  in arm_v7s_prot_to_pte()
343 	if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS))  in arm_v7s_prot_to_pte()
414 static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data,  in arm_v7s_init_pte()  argument
418 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_v7s_init_pte()
431 			tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg);  in arm_v7s_init_pte()
432 			if (WARN_ON(__arm_v7s_unmap(data, NULL, iova + i * sz,  in arm_v7s_init_pte()
434 				return -EINVAL;  in arm_v7s_init_pte()
438 			return -EEXIST;  in arm_v7s_init_pte()
459 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_v7s_install_table()
475 static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova,  in __arm_v7s_map()  argument
476 			 phys_addr_t paddr, size_t size, int prot,  in __arm_v7s_map()  argument
479 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_map()
481 	int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);  in __arm_v7s_map()
488 		return arm_v7s_init_pte(data, iova, paddr, prot,  in __arm_v7s_map()
493 		return -EINVAL;  in __arm_v7s_map()
498 		cptep = __arm_v7s_alloc_table(lvl + 1, gfp, data);  in __arm_v7s_map()
500 			return -ENOMEM;  in __arm_v7s_map()
504 			__arm_v7s_free_table(cptep, lvl + 1, data);  in __arm_v7s_map()
511 		cptep = iopte_deref(pte, lvl, data);  in __arm_v7s_map()
515 		return -EEXIST;  in __arm_v7s_map()
519 	return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp);  in __arm_v7s_map()
526 	struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops);  in arm_v7s_map_pages()  local
527 	int ret = -EINVAL;  in arm_v7s_map_pages()
529 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) ||  in arm_v7s_map_pages()
530 		    paddr >= (1ULL << data->iop.cfg.oas)))  in arm_v7s_map_pages()
531 		return -ERANGE;  in arm_v7s_map_pages()
537 	while (pgcount--) {  in arm_v7s_map_pages()
538 		ret = __arm_v7s_map(data, iova, paddr, pgsize, prot, 1, data->pgd,  in arm_v7s_map_pages()
558 		       phys_addr_t paddr, size_t size, int prot, gfp_t gfp)  in arm_v7s_map()  argument
560 	return arm_v7s_map_pages(ops, iova, paddr, size, 1, prot, gfp, NULL);  in arm_v7s_map()
565 	struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop);  in arm_v7s_free_pgtable()  local
568 	for (i = 0; i < ARM_V7S_PTES_PER_LVL(1, &data->iop.cfg); i++) {  in arm_v7s_free_pgtable()
569 		arm_v7s_iopte pte = data->pgd[i];  in arm_v7s_free_pgtable()
572 			__arm_v7s_free_table(iopte_deref(pte, 1, data),  in arm_v7s_free_pgtable()
573 					     2, data);  in arm_v7s_free_pgtable()
575 	__arm_v7s_free_table(data->pgd, 1, data);  in arm_v7s_free_pgtable()
576 	kmem_cache_destroy(data->l2_tables);  in arm_v7s_free_pgtable()
577 	kfree(data);  in arm_v7s_free_pgtable()
580 static arm_v7s_iopte arm_v7s_split_cont(struct arm_v7s_io_pgtable *data,  in arm_v7s_split_cont()  argument
584 	struct io_pgtable *iop = &data->iop;  in arm_v7s_split_cont()
586 	size_t size = ARM_V7S_BLOCK_SIZE(lvl);  in arm_v7s_split_cont()  local
594 	ptep -= idx & (ARM_V7S_CONT_PAGES - 1);  in arm_v7s_split_cont()
597 		ptep[i] = pte + i * size;  in arm_v7s_split_cont()
599 	__arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg);  in arm_v7s_split_cont()
601 	size *= ARM_V7S_CONT_PAGES;  in arm_v7s_split_cont()
602 	io_pgtable_tlb_flush_walk(iop, iova, size, size);  in arm_v7s_split_cont()
606 static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data,  in arm_v7s_split_blk_unmap()  argument
608 				      unsigned long iova, size_t size,  in arm_v7s_split_blk_unmap()  argument
612 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_v7s_split_blk_unmap()
616 	tablep = __arm_v7s_alloc_table(2, GFP_ATOMIC, data);  in arm_v7s_split_blk_unmap()
621 	num_entries = size >> ARM_V7S_LVL_SHIFT(2);  in arm_v7s_split_blk_unmap()
628 	for (i = 0; i < num_ptes; i += num_entries, pte += size) {  in arm_v7s_split_blk_unmap()
638 		__arm_v7s_free_table(tablep, 2, data);  in arm_v7s_split_blk_unmap()
643 		tablep = iopte_deref(pte, 1, data);  in arm_v7s_split_blk_unmap()
644 		return __arm_v7s_unmap(data, gather, iova, size, 2, tablep);  in arm_v7s_split_blk_unmap()
647 	io_pgtable_tlb_add_page(&data->iop, gather, iova, size);  in arm_v7s_split_blk_unmap()
648 	return size;  in arm_v7s_split_blk_unmap()
651 static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,  in __arm_v7s_unmap()  argument
653 			      unsigned long iova, size_t size, int lvl,  in __arm_v7s_unmap()  argument
657 	struct io_pgtable *iop = &data->iop;  in __arm_v7s_unmap()
658 	int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);  in __arm_v7s_unmap()
664 	idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg);  in __arm_v7s_unmap()
685 		spin_lock_irqsave(&data->split_lock, flags);  in __arm_v7s_unmap()
686 		pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep);  in __arm_v7s_unmap()
687 		spin_unlock_irqrestore(&data->split_lock, flags);  in __arm_v7s_unmap()
690 	/* If the size matches this level, we're in the right place */  in __arm_v7s_unmap()
694 		__arm_v7s_set_pte(ptep, 0, num_entries, &iop->cfg);  in __arm_v7s_unmap()
701 				ptep = iopte_deref(pte[i], lvl, data);  in __arm_v7s_unmap()
702 				__arm_v7s_free_table(ptep, lvl + 1, data);  in __arm_v7s_unmap()
708 		return size;  in __arm_v7s_unmap()
714 		return arm_v7s_split_blk_unmap(data, gather, iova, size, pte[0],  in __arm_v7s_unmap()
719 	ptep = iopte_deref(pte[0], lvl, data);  in __arm_v7s_unmap()
720 	return __arm_v7s_unmap(data, gather, iova, size, lvl + 1, ptep);  in __arm_v7s_unmap()
727 	struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops);  in arm_v7s_unmap_pages()  local
730 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias)))  in arm_v7s_unmap_pages()
733 	while (pgcount--) {  in arm_v7s_unmap_pages()
734 		ret = __arm_v7s_unmap(data, gather, iova, pgsize, 1, data->pgd);  in arm_v7s_unmap_pages()
746 			    size_t size, struct iommu_iotlb_gather *gather)  in arm_v7s_unmap()  argument
748 	return arm_v7s_unmap_pages(ops, iova, size, 1, gather);  in arm_v7s_unmap()
754 	struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops);  in arm_v7s_iova_to_phys()  local
755 	arm_v7s_iopte *ptep = data->pgd, pte;  in arm_v7s_iova_to_phys()
760 		ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg);  in arm_v7s_iova_to_phys()
762 		ptep = iopte_deref(pte, lvl, data);  in arm_v7s_iova_to_phys()
771 	return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask);  in arm_v7s_iova_to_phys()
777 	struct arm_v7s_io_pgtable *data;  in arm_v7s_alloc_pgtable()  local
779 	if (cfg->ias > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS))  in arm_v7s_alloc_pgtable()
782 	if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 35 : ARM_V7S_ADDR_BITS))  in arm_v7s_alloc_pgtable()
785 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |  in arm_v7s_alloc_pgtable()
791 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT &&  in arm_v7s_alloc_pgtable()
792 	    !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS))  in arm_v7s_alloc_pgtable()
795 	data = kmalloc(sizeof(*data), GFP_KERNEL);  in arm_v7s_alloc_pgtable()
796 	if (!data)  in arm_v7s_alloc_pgtable()
799 	spin_lock_init(&data->split_lock);  in arm_v7s_alloc_pgtable()
800 	data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2",  in arm_v7s_alloc_pgtable()
804 	if (!data->l2_tables)  in arm_v7s_alloc_pgtable()
807 	data->iop.ops = (struct io_pgtable_ops) {  in arm_v7s_alloc_pgtable()
816 	data->iop.cfg = *cfg;  in arm_v7s_alloc_pgtable()
822 	cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M;  in arm_v7s_alloc_pgtable()
825 	cfg->arm_v7s_cfg.tcr = 0;  in arm_v7s_alloc_pgtable()
829 	 * under the non-TEX-remap interpretation of those attribute bits,  in arm_v7s_alloc_pgtable()
830 	 * excepting various implementation-defined aspects of shareability.  in arm_v7s_alloc_pgtable()
832 	cfg->arm_v7s_cfg.prrr = ARM_V7S_PRRR_TR(1, ARM_V7S_PRRR_TYPE_DEVICE) |  in arm_v7s_alloc_pgtable()
837 	cfg->arm_v7s_cfg.nmrr = ARM_V7S_NMRR_IR(7, ARM_V7S_RGN_WBWA) |  in arm_v7s_alloc_pgtable()
841 	data->pgd = __arm_v7s_alloc_table(1, GFP_KERNEL, data);  in arm_v7s_alloc_pgtable()
842 	if (!data->pgd)  in arm_v7s_alloc_pgtable()
849 	cfg->arm_v7s_cfg.ttbr = virt_to_phys(data->pgd) | ARM_V7S_TTBR_S |  in arm_v7s_alloc_pgtable()
850 				(cfg->coherent_walk ? (ARM_V7S_TTBR_NOS |  in arm_v7s_alloc_pgtable()
855 	return &data->iop;  in arm_v7s_alloc_pgtable()
858 	kmem_cache_destroy(data->l2_tables);  in arm_v7s_alloc_pgtable()
859 	kfree(data);  in arm_v7s_alloc_pgtable()
877 static void __init dummy_tlb_flush(unsigned long iova, size_t size,  in dummy_tlb_flush()  argument
881 	WARN_ON(!(size & cfg_cookie->pgsize_bitmap));  in dummy_tlb_flush()
900 		-EFAULT;				\
914 	unsigned int iova, size, iova_start;  in arm_v7s_do_selftests()  local
924 		return -EINVAL;  in arm_v7s_do_selftests()
931 	if (ops->iova_to_phys(ops, 42))  in arm_v7s_do_selftests()
934 	if (ops->iova_to_phys(ops, SZ_1G + 42))  in arm_v7s_do_selftests()
937 	if (ops->iova_to_phys(ops, SZ_2G + 42))  in arm_v7s_do_selftests()
945 		size = 1UL << i;  in arm_v7s_do_selftests()
946 		if (ops->map(ops, iova, iova, size, IOMMU_READ |  in arm_v7s_do_selftests()
953 		if (!ops->map(ops, iova, iova + size, size,  in arm_v7s_do_selftests()
957 		if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_v7s_do_selftests()
966 	size = 1UL << __ffs(cfg.pgsize_bitmap);  in arm_v7s_do_selftests()
969 		if (ops->unmap(ops, iova_start + size, size, NULL) != size)  in arm_v7s_do_selftests()
973 		if (ops->map(ops, iova_start + size, size, size, IOMMU_READ, GFP_KERNEL))  in arm_v7s_do_selftests()
976 		if (ops->iova_to_phys(ops, iova_start + size + 42)  in arm_v7s_do_selftests()
977 		    != (size + 42))  in arm_v7s_do_selftests()
985 		size = 1UL << i;  in arm_v7s_do_selftests()
987 		if (ops->unmap(ops, iova, size, NULL) != size)  in arm_v7s_do_selftests()
990 		if (ops->iova_to_phys(ops, iova + 42))  in arm_v7s_do_selftests()
994 		if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))  in arm_v7s_do_selftests()
997 		if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_v7s_do_selftests()