1 // SPDX-License-Identifier: GPL-2.0-only
3  * CPU-agnostic ARM page table allocator.
5  * ARMv7 Short-descriptor format, supporting
6  * - Basic memory attributes
7  * - Simplified access permissions (AP[2:1] model)
8  * - Backwards-compatible TEX remap
9  * - Large pages/supersections (if indicated by the caller)
12  * - Legacy access permissions (AP[2:0] model)
15  * - PXN
16  * - Domains
18  * Copyright (C) 2014-2015 ARM Limited
19  * Copyright (c) 2014-2015 MediaTek Inc.
22 #define pr_fmt(fmt)	"arm-v7s io-pgtable: " fmt
25 #include <linux/dma-mapping.h>
27 #include <linux/io-pgtable.h>
51 #define _ARM_V7S_LVL_BITS(lvl, cfg)	((lvl) == 1 ? ((cfg)->ias - 20) : 8)
62 #define _ARM_V7S_IDX_MASK(lvl, cfg)	(ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1)
88 #define ARM_V7S_ATTR_XN(lvl)		BIT(4 * (2 - (lvl)))
100  * fields relative to that 8-bit block, plus a total shift relative to the PTE.
102 #define ARM_V7S_ATTR_SHIFT(lvl)		(16 - (lvl) * 6)
182 		(cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT);  in arm_v7s_is_mtk_enabled()
234 				  struct arm_v7s_io_pgtable *data)  in iopte_deref()  argument
236 	return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg));  in iopte_deref()
240 				   struct arm_v7s_io_pgtable *data)  in __arm_v7s_alloc_table()  argument
242 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_alloc_table()
243 	struct device *dev = cfg->iommu_dev;  in __arm_v7s_alloc_table()
246 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_alloc_table()  local
254 	gfp_l1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ?  in __arm_v7s_alloc_table()
258 		table = (void *)__get_free_pages(gfp_l1 | __GFP_ZERO, get_order(size));  in __arm_v7s_alloc_table()
260 		table = kmem_cache_zalloc(data->l2_tables, gfp);  in __arm_v7s_alloc_table()
266 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ?  in __arm_v7s_alloc_table()
267 	    phys >= (1ULL << cfg->oas) : phys != (arm_v7s_iopte)phys) {  in __arm_v7s_alloc_table()
272 	if (!cfg->coherent_walk) {  in __arm_v7s_alloc_table()
273 		dma = dma_map_single(dev, table, size, DMA_TO_DEVICE);  in __arm_v7s_alloc_table()
290 	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);  in __arm_v7s_alloc_table()
293 		free_pages((unsigned long)table, get_order(size));  in __arm_v7s_alloc_table()
295 		kmem_cache_free(data->l2_tables, table);  in __arm_v7s_alloc_table()
300 				 struct arm_v7s_io_pgtable *data)  in __arm_v7s_free_table()  argument
302 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_free_table()
303 	struct device *dev = cfg->iommu_dev;  in __arm_v7s_free_table()
304 	size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg);  in __arm_v7s_free_table()  local
306 	if (!cfg->coherent_walk)  in __arm_v7s_free_table()
307 		dma_unmap_single(dev, __arm_v7s_dma_addr(table), size,  in __arm_v7s_free_table()
310 		free_pages((unsigned long)table, get_order(size));  in __arm_v7s_free_table()
312 		kmem_cache_free(data->l2_tables, table);  in __arm_v7s_free_table()
318 	if (cfg->coherent_walk)  in __arm_v7s_pte_sync()
321 	dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep),  in __arm_v7s_pte_sync()
338 	bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS);  in arm_v7s_prot_to_pte()
360 	if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS))  in arm_v7s_prot_to_pte()
431 static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data,  in arm_v7s_init_pte()  argument
435 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_v7s_init_pte()
448 			tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg);  in arm_v7s_init_pte()
449 			if (WARN_ON(__arm_v7s_unmap(data, NULL, iova + i * sz,  in arm_v7s_init_pte()
451 				return -EINVAL;  in arm_v7s_init_pte()
455 			return -EEXIST;  in arm_v7s_init_pte()
478 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT)  in arm_v7s_install_table()
481 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_v7s_install_table()
497 static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova,  in __arm_v7s_map()  argument
498 			 phys_addr_t paddr, size_t size, int prot,  in __arm_v7s_map()  argument
501 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_v7s_map()
503 	int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);  in __arm_v7s_map()
510 		return arm_v7s_init_pte(data, iova, paddr, prot,  in __arm_v7s_map()
515 		return -EINVAL;  in __arm_v7s_map()
520 		cptep = __arm_v7s_alloc_table(lvl + 1, gfp, data);  in __arm_v7s_map()
522 			return -ENOMEM;  in __arm_v7s_map()
526 			__arm_v7s_free_table(cptep, lvl + 1, data);  in __arm_v7s_map()
533 		cptep = iopte_deref(pte, lvl, data);  in __arm_v7s_map()
537 		return -EEXIST;  in __arm_v7s_map()
541 	return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp);  in __arm_v7s_map()
548 	struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops);  in arm_v7s_map_pages()  local
549 	int ret = -EINVAL;  in arm_v7s_map_pages()
551 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) ||  in arm_v7s_map_pages()
552 		    paddr >= (1ULL << data->iop.cfg.oas)))  in arm_v7s_map_pages()
553 		return -ERANGE;  in arm_v7s_map_pages()
559 	while (pgcount--) {  in arm_v7s_map_pages()
560 		ret = __arm_v7s_map(data, iova, paddr, pgsize, prot, 1, data->pgd,  in arm_v7s_map_pages()
580 		       phys_addr_t paddr, size_t size, int prot, gfp_t gfp)  in arm_v7s_map()  argument
582 	return arm_v7s_map_pages(ops, iova, paddr, size, 1, prot, gfp, NULL);  in arm_v7s_map()
587 	struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop);  in arm_v7s_free_pgtable()  local
590 	for (i = 0; i < ARM_V7S_PTES_PER_LVL(1, &data->iop.cfg); i++) {  in arm_v7s_free_pgtable()
591 		arm_v7s_iopte pte = data->pgd[i];  in arm_v7s_free_pgtable()
594 			__arm_v7s_free_table(iopte_deref(pte, 1, data),  in arm_v7s_free_pgtable()
595 					     2, data);  in arm_v7s_free_pgtable()
597 	__arm_v7s_free_table(data->pgd, 1, data);  in arm_v7s_free_pgtable()
598 	kmem_cache_destroy(data->l2_tables);  in arm_v7s_free_pgtable()
599 	kfree(data);  in arm_v7s_free_pgtable()
602 static arm_v7s_iopte arm_v7s_split_cont(struct arm_v7s_io_pgtable *data,  in arm_v7s_split_cont()  argument
606 	struct io_pgtable *iop = &data->iop;  in arm_v7s_split_cont()
608 	size_t size = ARM_V7S_BLOCK_SIZE(lvl);  in arm_v7s_split_cont()  local
616 	ptep -= idx & (ARM_V7S_CONT_PAGES - 1);  in arm_v7s_split_cont()
619 		ptep[i] = pte + i * size;  in arm_v7s_split_cont()
621 	__arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg);  in arm_v7s_split_cont()
623 	size *= ARM_V7S_CONT_PAGES;  in arm_v7s_split_cont()
624 	io_pgtable_tlb_flush_walk(iop, iova, size, size);  in arm_v7s_split_cont()
628 static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data,  in arm_v7s_split_blk_unmap()  argument
630 				      unsigned long iova, size_t size,  in arm_v7s_split_blk_unmap()  argument
634 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_v7s_split_blk_unmap()
638 	tablep = __arm_v7s_alloc_table(2, GFP_ATOMIC, data);  in arm_v7s_split_blk_unmap()
643 	num_entries = size >> ARM_V7S_LVL_SHIFT(2);  in arm_v7s_split_blk_unmap()
650 	for (i = 0; i < num_ptes; i += num_entries, pte += size) {  in arm_v7s_split_blk_unmap()
660 		__arm_v7s_free_table(tablep, 2, data);  in arm_v7s_split_blk_unmap()
665 		tablep = iopte_deref(pte, 1, data);  in arm_v7s_split_blk_unmap()
666 		return __arm_v7s_unmap(data, gather, iova, size, 2, tablep);  in arm_v7s_split_blk_unmap()
669 	io_pgtable_tlb_add_page(&data->iop, gather, iova, size);  in arm_v7s_split_blk_unmap()
670 	return size;  in arm_v7s_split_blk_unmap()
673 static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,  in __arm_v7s_unmap()  argument
675 			      unsigned long iova, size_t size, int lvl,  in __arm_v7s_unmap()  argument
679 	struct io_pgtable *iop = &data->iop;  in __arm_v7s_unmap()
680 	int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);  in __arm_v7s_unmap()
686 	idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg);  in __arm_v7s_unmap()
707 		spin_lock_irqsave(&data->split_lock, flags);  in __arm_v7s_unmap()
708 		pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep);  in __arm_v7s_unmap()
709 		spin_unlock_irqrestore(&data->split_lock, flags);  in __arm_v7s_unmap()
712 	/* If the size matches this level, we're in the right place */  in __arm_v7s_unmap()
716 		__arm_v7s_set_pte(ptep, 0, num_entries, &iop->cfg);  in __arm_v7s_unmap()
723 				ptep = iopte_deref(pte[i], lvl, data);  in __arm_v7s_unmap()
724 				__arm_v7s_free_table(ptep, lvl + 1, data);  in __arm_v7s_unmap()
730 		return size;  in __arm_v7s_unmap()
736 		return arm_v7s_split_blk_unmap(data, gather, iova, size, pte[0],  in __arm_v7s_unmap()
741 	ptep = iopte_deref(pte[0], lvl, data);  in __arm_v7s_unmap()
742 	return __arm_v7s_unmap(data, gather, iova, size, lvl + 1, ptep);  in __arm_v7s_unmap()
749 	struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops);  in arm_v7s_unmap_pages()  local
752 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias)))  in arm_v7s_unmap_pages()
755 	while (pgcount--) {  in arm_v7s_unmap_pages()
756 		ret = __arm_v7s_unmap(data, gather, iova, pgsize, 1, data->pgd);  in arm_v7s_unmap_pages()
768 			    size_t size, struct iommu_iotlb_gather *gather)  in arm_v7s_unmap()  argument
770 	return arm_v7s_unmap_pages(ops, iova, size, 1, gather);  in arm_v7s_unmap()
776 	struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops);  in arm_v7s_iova_to_phys()  local
777 	arm_v7s_iopte *ptep = data->pgd, pte;  in arm_v7s_iova_to_phys()
782 		ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg);  in arm_v7s_iova_to_phys()
784 		ptep = iopte_deref(pte, lvl, data);  in arm_v7s_iova_to_phys()
793 	return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask);  in arm_v7s_iova_to_phys()
799 	struct arm_v7s_io_pgtable *data;  in arm_v7s_alloc_pgtable()  local
803 	if (cfg->ias > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS))  in arm_v7s_alloc_pgtable()
806 	if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 35 : ARM_V7S_ADDR_BITS))  in arm_v7s_alloc_pgtable()
809 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |  in arm_v7s_alloc_pgtable()
816 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT &&  in arm_v7s_alloc_pgtable()
817 	    !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS))  in arm_v7s_alloc_pgtable()
820 	if ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) &&  in arm_v7s_alloc_pgtable()
824 	data = kmalloc(sizeof(*data), GFP_KERNEL);  in arm_v7s_alloc_pgtable()
825 	if (!data)  in arm_v7s_alloc_pgtable()
828 	spin_lock_init(&data->split_lock);  in arm_v7s_alloc_pgtable()
834 	slab_flag = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ?  in arm_v7s_alloc_pgtable()
837 	data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2",  in arm_v7s_alloc_pgtable()
841 	if (!data->l2_tables)  in arm_v7s_alloc_pgtable()
844 	data->iop.ops = (struct io_pgtable_ops) {  in arm_v7s_alloc_pgtable()
853 	data->iop.cfg = *cfg;  in arm_v7s_alloc_pgtable()
859 	cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M;  in arm_v7s_alloc_pgtable()
862 	cfg->arm_v7s_cfg.tcr = 0;  in arm_v7s_alloc_pgtable()
866 	 * under the non-TEX-remap interpretation of those attribute bits,  in arm_v7s_alloc_pgtable()
867 	 * excepting various implementation-defined aspects of shareability.  in arm_v7s_alloc_pgtable()
869 	cfg->arm_v7s_cfg.prrr = ARM_V7S_PRRR_TR(1, ARM_V7S_PRRR_TYPE_DEVICE) |  in arm_v7s_alloc_pgtable()
874 	cfg->arm_v7s_cfg.nmrr = ARM_V7S_NMRR_IR(7, ARM_V7S_RGN_WBWA) |  in arm_v7s_alloc_pgtable()
878 	data->pgd = __arm_v7s_alloc_table(1, GFP_KERNEL, data);  in arm_v7s_alloc_pgtable()
879 	if (!data->pgd)  in arm_v7s_alloc_pgtable()
886 	paddr = virt_to_phys(data->pgd);  in arm_v7s_alloc_pgtable()
888 		cfg->arm_v7s_cfg.ttbr = paddr | upper_32_bits(paddr);  in arm_v7s_alloc_pgtable()
890 		cfg->arm_v7s_cfg.ttbr = paddr | ARM_V7S_TTBR_S |  in arm_v7s_alloc_pgtable()
891 					(cfg->coherent_walk ? (ARM_V7S_TTBR_NOS |  in arm_v7s_alloc_pgtable()
896 	return &data->iop;  in arm_v7s_alloc_pgtable()
899 	kmem_cache_destroy(data->l2_tables);  in arm_v7s_alloc_pgtable()
900 	kfree(data);  in arm_v7s_alloc_pgtable()
918 static void __init dummy_tlb_flush(unsigned long iova, size_t size,  in dummy_tlb_flush()  argument
922 	WARN_ON(!(size & cfg_cookie->pgsize_bitmap));  in dummy_tlb_flush()
941 		-EFAULT;				\
955 	unsigned int iova, size, iova_start;  in arm_v7s_do_selftests()  local
965 		return -EINVAL;  in arm_v7s_do_selftests()
972 	if (ops->iova_to_phys(ops, 42))  in arm_v7s_do_selftests()
975 	if (ops->iova_to_phys(ops, SZ_1G + 42))  in arm_v7s_do_selftests()
978 	if (ops->iova_to_phys(ops, SZ_2G + 42))  in arm_v7s_do_selftests()
986 		size = 1UL << i;  in arm_v7s_do_selftests()
987 		if (ops->map(ops, iova, iova, size, IOMMU_READ |  in arm_v7s_do_selftests()
994 		if (!ops->map(ops, iova, iova + size, size,  in arm_v7s_do_selftests()
998 		if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_v7s_do_selftests()
1007 	size = 1UL << __ffs(cfg.pgsize_bitmap);  in arm_v7s_do_selftests()
1010 		if (ops->unmap(ops, iova_start + size, size, NULL) != size)  in arm_v7s_do_selftests()
1014 		if (ops->map(ops, iova_start + size, size, size, IOMMU_READ, GFP_KERNEL))  in arm_v7s_do_selftests()
1017 		if (ops->iova_to_phys(ops, iova_start + size + 42)  in arm_v7s_do_selftests()
1018 		    != (size + 42))  in arm_v7s_do_selftests()
1026 		size = 1UL << i;  in arm_v7s_do_selftests()
1028 		if (ops->unmap(ops, iova, size, NULL) != size)  in arm_v7s_do_selftests()
1031 		if (ops->iova_to_phys(ops, iova + 42))  in arm_v7s_do_selftests()
1035 		if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))  in arm_v7s_do_selftests()
1038 		if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_v7s_do_selftests()