Lines Matching +full:bit +full:- +full:manipulation
1 // SPDX-License-Identifier: GPL-2.0
3 * intel-pasid.c - PASID idr, table and entry manipulation
18 #include <linux/pci-ats.h>
36 raw_spin_lock_irqsave(&iommu->register_lock, flags); in vcmd_alloc_pasid()
37 dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC); in vcmd_alloc_pasid()
40 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in vcmd_alloc_pasid()
48 pr_info("IOMMU: %s: No PASID available\n", iommu->name); in vcmd_alloc_pasid()
49 ret = -ENOSPC; in vcmd_alloc_pasid()
52 ret = -ENODEV; in vcmd_alloc_pasid()
54 iommu->name, status_code); in vcmd_alloc_pasid()
66 raw_spin_lock_irqsave(&iommu->register_lock, flags); in vcmd_free_pasid()
67 dmar_writeq(iommu->reg + DMAR_VCMD_REG, in vcmd_free_pasid()
71 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in vcmd_free_pasid()
78 pr_info("IOMMU: %s: Invalid PASID\n", iommu->name); in vcmd_free_pasid()
82 iommu->name, status_code); in vcmd_free_pasid()
92 * single-thread context.
105 return -ENODEV; in intel_pasid_alloc_table()
106 if (WARN_ON(info->pasid_table)) in intel_pasid_alloc_table()
107 return -EEXIST; in intel_pasid_alloc_table()
111 return -ENOMEM; in intel_pasid_alloc_table()
113 if (info->pasid_supported) in intel_pasid_alloc_table()
117 size = max_pasid >> (PASID_PDE_SHIFT - 3); in intel_pasid_alloc_table()
119 pages = alloc_pages_node(info->iommu->node, in intel_pasid_alloc_table()
123 return -ENOMEM; in intel_pasid_alloc_table()
126 pasid_table->table = page_address(pages); in intel_pasid_alloc_table()
127 pasid_table->order = order; in intel_pasid_alloc_table()
128 pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3); in intel_pasid_alloc_table()
129 info->pasid_table = pasid_table; in intel_pasid_alloc_table()
131 if (!ecap_coherent(info->iommu->ecap)) in intel_pasid_alloc_table()
132 clflush_cache_range(pasid_table->table, (1 << order) * PAGE_SIZE); in intel_pasid_alloc_table()
146 if (!info || !dev_is_pci(dev) || !info->pasid_table) in intel_pasid_free_table()
149 pasid_table = info->pasid_table; in intel_pasid_free_table()
150 info->pasid_table = NULL; in intel_pasid_free_table()
153 dir = pasid_table->table; in intel_pasid_free_table()
154 max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT; in intel_pasid_free_table()
160 free_pages((unsigned long)pasid_table->table, pasid_table->order); in intel_pasid_free_table()
172 return info->pasid_table; in intel_pasid_get_table()
180 if (!info || !info->pasid_table) in intel_pasid_get_dev_max_id()
183 return info->pasid_table->max_pasid; in intel_pasid_get_dev_max_id()
198 dir = pasid_table->table; in intel_pasid_get_entry()
206 entries = alloc_pgtable_page(info->iommu->node, GFP_ATOMIC); in intel_pasid_get_entry()
221 if (!ecap_coherent(info->iommu->ecap)) { in intel_pasid_get_entry()
231 * Interfaces for PASID table entry manipulation:
235 WRITE_ONCE(pe->val[0], 0); in pasid_clear_entry()
236 WRITE_ONCE(pe->val[1], 0); in pasid_clear_entry()
237 WRITE_ONCE(pe->val[2], 0); in pasid_clear_entry()
238 WRITE_ONCE(pe->val[3], 0); in pasid_clear_entry()
239 WRITE_ONCE(pe->val[4], 0); in pasid_clear_entry()
240 WRITE_ONCE(pe->val[5], 0); in pasid_clear_entry()
241 WRITE_ONCE(pe->val[6], 0); in pasid_clear_entry()
242 WRITE_ONCE(pe->val[7], 0); in pasid_clear_entry()
247 WRITE_ONCE(pe->val[0], PASID_PTE_FPD); in pasid_clear_entry_with_fpd()
248 WRITE_ONCE(pe->val[1], 0); in pasid_clear_entry_with_fpd()
249 WRITE_ONCE(pe->val[2], 0); in pasid_clear_entry_with_fpd()
250 WRITE_ONCE(pe->val[3], 0); in pasid_clear_entry_with_fpd()
251 WRITE_ONCE(pe->val[4], 0); in pasid_clear_entry_with_fpd()
252 WRITE_ONCE(pe->val[5], 0); in pasid_clear_entry_with_fpd()
253 WRITE_ONCE(pe->val[6], 0); in pasid_clear_entry_with_fpd()
254 WRITE_ONCE(pe->val[7], 0); in pasid_clear_entry_with_fpd()
281 * Setup the DID(Domain Identifier) field (Bit 64~79) of scalable mode
287 pasid_set_bits(&pe->val[1], GENMASK_ULL(15, 0), value); in pasid_set_domain_id()
296 return (u16)(READ_ONCE(pe->val[1]) & GENMASK_ULL(15, 0)); in pasid_get_domain_id()
300 * Setup the SLPTPTR(Second Level Page Table Pointer) field (Bit 12~63)
306 pasid_set_bits(&pe->val[0], VTD_PAGE_MASK, value); in pasid_set_slptr()
310 * Setup the AW(Address Width) field (Bit 2~4) of a scalable mode PASID
316 pasid_set_bits(&pe->val[0], GENMASK_ULL(4, 2), value << 2); in pasid_set_address_width()
320 * Setup the PGTT(PASID Granular Translation Type) field (Bit 6~8)
326 pasid_set_bits(&pe->val[0], GENMASK_ULL(8, 6), value << 6); in pasid_set_translation_type()
331 * Disable) field (Bit 1) of a scalable mode PASID entry.
335 pasid_set_bits(&pe->val[0], 1 << 1, 0); in pasid_set_fault_enable()
339 * Setup the WPE(Write Protect Enable) field (Bit 132) of a
344 pasid_set_bits(&pe->val[2], 1 << 4, 1 << 4); in pasid_set_wpe()
348 * Setup the P(Present) field (Bit 0) of a scalable mode PASID
353 pasid_set_bits(&pe->val[0], 1 << 0, 1); in pasid_set_present()
357 * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID
362 pasid_set_bits(&pe->val[1], 1 << 23, value << 23); in pasid_set_page_snoop()
366 * Setup No Execute Enable bit (Bit 133) of a scalable mode PASID
367 * entry. It is required when XD bit of the first level page table
372 pasid_set_bits(&pe->val[2], 1 << 5, 1 << 5); in pasid_set_nxe()
376 * Setup the Page Snoop (PGSNP) field (Bit 88) of a scalable mode
382 pasid_set_bits(&pe->val[1], 1ULL << 24, 1ULL << 24); in pasid_set_pgsnp()
386 * Setup the First Level Page table Pointer field (Bit 140~191)
392 pasid_set_bits(&pe->val[2], VTD_PAGE_MASK, value); in pasid_set_flptr()
396 * Setup the First Level Paging Mode field (Bit 130~131) of a
402 pasid_set_bits(&pe->val[2], GENMASK_ULL(3, 2), value << 2); in pasid_set_flpm()
428 if (!info || !info->ats_enabled) in devtlb_invalidation_with_pasid()
431 sid = info->bus << 8 | info->devfn; in devtlb_invalidation_with_pasid()
432 qdep = info->ats_qdep; in devtlb_invalidation_with_pasid()
433 pfsid = info->pfsid; in devtlb_invalidation_with_pasid()
437 * devTLB flush w/o PASID should be used. For non-zero PASID under in devtlb_invalidation_with_pasid()
442 qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT); in devtlb_invalidation_with_pasid()
444 qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT); in devtlb_invalidation_with_pasid()
453 spin_lock(&iommu->lock); in intel_pasid_tear_down_entry()
456 spin_unlock(&iommu->lock); in intel_pasid_tear_down_entry()
463 spin_unlock(&iommu->lock); in intel_pasid_tear_down_entry()
465 if (!ecap_coherent(iommu->ecap)) in intel_pasid_tear_down_entry()
471 qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); in intel_pasid_tear_down_entry()
473 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); in intel_pasid_tear_down_entry()
476 if (!cap_caching_mode(iommu->cap)) in intel_pasid_tear_down_entry()
482 * Caller of it should not modify the in-use pasid table entries.
488 if (!ecap_coherent(iommu->ecap)) in pasid_flush_caches()
491 if (cap_caching_mode(iommu->cap)) { in pasid_flush_caches()
493 qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); in pasid_flush_caches()
509 if (!ecap_flts(iommu->ecap)) { in intel_pasid_setup_first_level()
511 iommu->name); in intel_pasid_setup_first_level()
512 return -EINVAL; in intel_pasid_setup_first_level()
515 if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) { in intel_pasid_setup_first_level()
516 pr_err("No 5-level paging support for first-level on %s\n", in intel_pasid_setup_first_level()
517 iommu->name); in intel_pasid_setup_first_level()
518 return -EINVAL; in intel_pasid_setup_first_level()
521 spin_lock(&iommu->lock); in intel_pasid_setup_first_level()
524 spin_unlock(&iommu->lock); in intel_pasid_setup_first_level()
525 return -ENODEV; in intel_pasid_setup_first_level()
529 spin_unlock(&iommu->lock); in intel_pasid_setup_first_level()
530 return -EBUSY; in intel_pasid_setup_first_level()
545 pasid_set_address_width(pte, iommu->agaw); in intel_pasid_setup_first_level()
546 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); in intel_pasid_setup_first_level()
552 spin_unlock(&iommu->lock); in intel_pasid_setup_first_level()
569 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { in iommu_skip_agaw()
572 return -EINVAL; in iommu_skip_agaw()
595 if (!ecap_slts(iommu->ecap)) { in intel_pasid_setup_second_level()
597 iommu->name); in intel_pasid_setup_second_level()
598 return -EINVAL; in intel_pasid_setup_second_level()
601 pgd = domain->pgd; in intel_pasid_setup_second_level()
605 return -EINVAL; in intel_pasid_setup_second_level()
611 spin_lock(&iommu->lock); in intel_pasid_setup_second_level()
614 spin_unlock(&iommu->lock); in intel_pasid_setup_second_level()
615 return -ENODEV; in intel_pasid_setup_second_level()
619 spin_unlock(&iommu->lock); in intel_pasid_setup_second_level()
620 return -EBUSY; in intel_pasid_setup_second_level()
629 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); in intel_pasid_setup_second_level()
632 spin_unlock(&iommu->lock); in intel_pasid_setup_second_level()
649 spin_lock(&iommu->lock); in intel_pasid_setup_pass_through()
652 spin_unlock(&iommu->lock); in intel_pasid_setup_pass_through()
653 return -ENODEV; in intel_pasid_setup_pass_through()
657 spin_unlock(&iommu->lock); in intel_pasid_setup_pass_through()
658 return -EBUSY; in intel_pasid_setup_pass_through()
663 pasid_set_address_width(pte, iommu->agaw); in intel_pasid_setup_pass_through()
666 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); in intel_pasid_setup_pass_through()
668 spin_unlock(&iommu->lock); in intel_pasid_setup_pass_through()
684 spin_lock(&iommu->lock); in intel_pasid_setup_page_snoop_control()
687 spin_unlock(&iommu->lock); in intel_pasid_setup_page_snoop_control()
693 spin_unlock(&iommu->lock); in intel_pasid_setup_page_snoop_control()
695 if (!ecap_coherent(iommu->ecap)) in intel_pasid_setup_page_snoop_control()
699 * VT-d spec 3.4 table23 states guides for cache invalidation: in intel_pasid_setup_page_snoop_control()
701 * - PASID-selective-within-Domain PASID-cache invalidation in intel_pasid_setup_page_snoop_control()
702 * - PASID-selective PASID-based IOTLB invalidation in intel_pasid_setup_page_snoop_control()
703 * - If (pasid is RID_PASID) in intel_pasid_setup_page_snoop_control()
704 * - Global Device-TLB invalidation to affected functions in intel_pasid_setup_page_snoop_control()
706 * - PASID-based Device-TLB invalidation (with S=1 and in intel_pasid_setup_page_snoop_control()
710 qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); in intel_pasid_setup_page_snoop_control()
713 if (!cap_caching_mode(iommu->cap)) in intel_pasid_setup_page_snoop_control()