1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37
38 #include <linux/fsl/mc.h>
39
40 #include "arm-smmu.h"
41 #include "../../dma-iommu.h"
42
43 /*
44 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45 * global register space are still, in fact, using a hypervisor to mediate it
46 * by trapping and emulating register accesses. Sadly, some deployed versions
47 * of said trapping code have bugs wherein they go horribly wrong for stores
48 * using r31 (i.e. XZR/WZR) as the source register.
49 */
50 #define QCOM_DUMMY_VAL -1
51
52 #define MSI_IOVA_BASE 0x8000000
53 #define MSI_IOVA_LENGTH 0x100000
54
55 static int force_stage;
56 module_param(force_stage, int, S_IRUGO);
57 MODULE_PARM_DESC(force_stage,
58 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59 static bool disable_bypass =
60 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61 module_param(disable_bypass, bool, S_IRUGO);
62 MODULE_PARM_DESC(disable_bypass,
63 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
64
65 #define s2cr_init_val (struct arm_smmu_s2cr){ \
66 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
67 }
68
69 static bool using_legacy_binding, using_generic_binding;
70
arm_smmu_rpm_get(struct arm_smmu_device * smmu)71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
72 {
73 if (pm_runtime_enabled(smmu->dev))
74 return pm_runtime_resume_and_get(smmu->dev);
75
76 return 0;
77 }
78
arm_smmu_rpm_put(struct arm_smmu_device * smmu)79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
80 {
81 if (pm_runtime_enabled(smmu->dev))
82 pm_runtime_put_autosuspend(smmu->dev);
83 }
84
to_smmu_domain(struct iommu_domain * dom)85 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
86 {
87 return container_of(dom, struct arm_smmu_domain, domain);
88 }
89
90 static struct platform_driver arm_smmu_driver;
91 static struct iommu_ops arm_smmu_ops;
92
93 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)94 static struct device_node *dev_get_dev_node(struct device *dev)
95 {
96 if (dev_is_pci(dev)) {
97 struct pci_bus *bus = to_pci_dev(dev)->bus;
98
99 while (!pci_is_root_bus(bus))
100 bus = bus->parent;
101 return of_node_get(bus->bridge->parent->of_node);
102 }
103
104 return of_node_get(dev->of_node);
105 }
106
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)107 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
108 {
109 *((__be32 *)data) = cpu_to_be32(alias);
110 return 0; /* Continue walking */
111 }
112
__find_legacy_master_phandle(struct device * dev,void * data)113 static int __find_legacy_master_phandle(struct device *dev, void *data)
114 {
115 struct of_phandle_iterator *it = *(void **)data;
116 struct device_node *np = it->node;
117 int err;
118
119 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
120 "#stream-id-cells", -1)
121 if (it->node == np) {
122 *(void **)data = dev;
123 return 1;
124 }
125 it->node = np;
126 return err == -ENOENT ? 0 : err;
127 }
128
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)129 static int arm_smmu_register_legacy_master(struct device *dev,
130 struct arm_smmu_device **smmu)
131 {
132 struct device *smmu_dev;
133 struct device_node *np;
134 struct of_phandle_iterator it;
135 void *data = ⁢
136 u32 *sids;
137 __be32 pci_sid;
138 int err;
139
140 np = dev_get_dev_node(dev);
141 if (!np || !of_property_present(np, "#stream-id-cells")) {
142 of_node_put(np);
143 return -ENODEV;
144 }
145
146 it.node = np;
147 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
148 __find_legacy_master_phandle);
149 smmu_dev = data;
150 of_node_put(np);
151 if (err == 0)
152 return -ENODEV;
153 if (err < 0)
154 return err;
155
156 if (dev_is_pci(dev)) {
157 /* "mmu-masters" assumes Stream ID == Requester ID */
158 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
159 &pci_sid);
160 it.cur = &pci_sid;
161 it.cur_count = 1;
162 }
163
164 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
165 &arm_smmu_ops);
166 if (err)
167 return err;
168
169 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
170 if (!sids)
171 return -ENOMEM;
172
173 *smmu = dev_get_drvdata(smmu_dev);
174 of_phandle_iterator_args(&it, sids, it.cur_count);
175 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
176 kfree(sids);
177 return err;
178 }
179 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)180 static int arm_smmu_register_legacy_master(struct device *dev,
181 struct arm_smmu_device **smmu)
182 {
183 return -ENODEV;
184 }
185 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
186
__arm_smmu_free_bitmap(unsigned long * map,int idx)187 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
188 {
189 clear_bit(idx, map);
190 }
191
192 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)193 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
194 int sync, int status)
195 {
196 unsigned int spin_cnt, delay;
197 u32 reg;
198
199 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
200 return smmu->impl->tlb_sync(smmu, page, sync, status);
201
202 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
203 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
204 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
205 reg = arm_smmu_readl(smmu, page, status);
206 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
207 return;
208 cpu_relax();
209 }
210 udelay(delay);
211 }
212 dev_err_ratelimited(smmu->dev,
213 "TLB sync timed out -- SMMU may be deadlocked\n");
214 }
215
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)216 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
217 {
218 unsigned long flags;
219
220 spin_lock_irqsave(&smmu->global_sync_lock, flags);
221 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
222 ARM_SMMU_GR0_sTLBGSTATUS);
223 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
224 }
225
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)226 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
227 {
228 struct arm_smmu_device *smmu = smmu_domain->smmu;
229 unsigned long flags;
230
231 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
232 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
233 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
234 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
235 }
236
arm_smmu_tlb_inv_context_s1(void * cookie)237 static void arm_smmu_tlb_inv_context_s1(void *cookie)
238 {
239 struct arm_smmu_domain *smmu_domain = cookie;
240 /*
241 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
242 * current CPU are visible beforehand.
243 */
244 wmb();
245 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
246 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
247 arm_smmu_tlb_sync_context(smmu_domain);
248 }
249
arm_smmu_tlb_inv_context_s2(void * cookie)250 static void arm_smmu_tlb_inv_context_s2(void *cookie)
251 {
252 struct arm_smmu_domain *smmu_domain = cookie;
253 struct arm_smmu_device *smmu = smmu_domain->smmu;
254
255 /* See above */
256 wmb();
257 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
258 arm_smmu_tlb_sync_global(smmu);
259 }
260
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)261 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
262 size_t granule, void *cookie, int reg)
263 {
264 struct arm_smmu_domain *smmu_domain = cookie;
265 struct arm_smmu_device *smmu = smmu_domain->smmu;
266 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
267 int idx = cfg->cbndx;
268
269 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
270 wmb();
271
272 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
273 iova = (iova >> 12) << 12;
274 iova |= cfg->asid;
275 do {
276 arm_smmu_cb_write(smmu, idx, reg, iova);
277 iova += granule;
278 } while (size -= granule);
279 } else {
280 iova >>= 12;
281 iova |= (u64)cfg->asid << 48;
282 do {
283 arm_smmu_cb_writeq(smmu, idx, reg, iova);
284 iova += granule >> 12;
285 } while (size -= granule);
286 }
287 }
288
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)289 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
290 size_t granule, void *cookie, int reg)
291 {
292 struct arm_smmu_domain *smmu_domain = cookie;
293 struct arm_smmu_device *smmu = smmu_domain->smmu;
294 int idx = smmu_domain->cfg.cbndx;
295
296 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
297 wmb();
298
299 iova >>= 12;
300 do {
301 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
302 arm_smmu_cb_writeq(smmu, idx, reg, iova);
303 else
304 arm_smmu_cb_write(smmu, idx, reg, iova);
305 iova += granule >> 12;
306 } while (size -= granule);
307 }
308
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)309 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
310 size_t granule, void *cookie)
311 {
312 struct arm_smmu_domain *smmu_domain = cookie;
313 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
314
315 if (cfg->flush_walk_prefer_tlbiasid) {
316 arm_smmu_tlb_inv_context_s1(cookie);
317 } else {
318 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
319 ARM_SMMU_CB_S1_TLBIVA);
320 arm_smmu_tlb_sync_context(cookie);
321 }
322 }
323
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)324 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
325 unsigned long iova, size_t granule,
326 void *cookie)
327 {
328 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
329 ARM_SMMU_CB_S1_TLBIVAL);
330 }
331
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)332 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
333 size_t granule, void *cookie)
334 {
335 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
336 ARM_SMMU_CB_S2_TLBIIPAS2);
337 arm_smmu_tlb_sync_context(cookie);
338 }
339
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)340 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
341 unsigned long iova, size_t granule,
342 void *cookie)
343 {
344 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
345 ARM_SMMU_CB_S2_TLBIIPAS2L);
346 }
347
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)348 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
349 size_t granule, void *cookie)
350 {
351 arm_smmu_tlb_inv_context_s2(cookie);
352 }
353 /*
354 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
355 * almost negligible, but the benefit of getting the first one in as far ahead
356 * of the sync as possible is significant, hence we don't just make this a
357 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
358 * think.
359 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)360 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
361 unsigned long iova, size_t granule,
362 void *cookie)
363 {
364 struct arm_smmu_domain *smmu_domain = cookie;
365 struct arm_smmu_device *smmu = smmu_domain->smmu;
366
367 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
368 wmb();
369
370 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
371 }
372
373 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
374 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
375 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
376 .tlb_add_page = arm_smmu_tlb_add_page_s1,
377 };
378
379 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
380 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
381 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
382 .tlb_add_page = arm_smmu_tlb_add_page_s2,
383 };
384
385 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
386 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
387 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
388 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
389 };
390
arm_smmu_context_fault(int irq,void * dev)391 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
392 {
393 u32 fsr, fsynr, cbfrsynra;
394 unsigned long iova;
395 struct iommu_domain *domain = dev;
396 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
397 struct arm_smmu_device *smmu = smmu_domain->smmu;
398 int idx = smmu_domain->cfg.cbndx;
399 int ret;
400
401 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
402 if (!(fsr & ARM_SMMU_FSR_FAULT))
403 return IRQ_NONE;
404
405 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
406 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
407 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
408
409 ret = report_iommu_fault(domain, NULL, iova,
410 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
411
412 if (ret == -ENOSYS)
413 dev_err_ratelimited(smmu->dev,
414 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
415 fsr, iova, fsynr, cbfrsynra, idx);
416
417 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
418 return IRQ_HANDLED;
419 }
420
arm_smmu_global_fault(int irq,void * dev)421 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
422 {
423 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
424 struct arm_smmu_device *smmu = dev;
425 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
426 DEFAULT_RATELIMIT_BURST);
427
428 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
429 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
430 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
431 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
432
433 if (!gfsr)
434 return IRQ_NONE;
435
436 if (__ratelimit(&rs)) {
437 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
438 (gfsr & ARM_SMMU_sGFSR_USF))
439 dev_err(smmu->dev,
440 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
441 (u16)gfsynr1);
442 else
443 dev_err(smmu->dev,
444 "Unexpected global fault, this could be serious\n");
445 dev_err(smmu->dev,
446 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
447 gfsr, gfsynr0, gfsynr1, gfsynr2);
448 }
449
450 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
451 return IRQ_HANDLED;
452 }
453
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)454 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
455 struct io_pgtable_cfg *pgtbl_cfg)
456 {
457 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
458 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
459 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
460
461 cb->cfg = cfg;
462
463 /* TCR */
464 if (stage1) {
465 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
466 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
467 } else {
468 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
469 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
470 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
471 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
472 else
473 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
474 }
475 } else {
476 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
477 }
478
479 /* TTBRs */
480 if (stage1) {
481 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
482 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
483 cb->ttbr[1] = 0;
484 } else {
485 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
486 cfg->asid);
487 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
488 cfg->asid);
489
490 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
491 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
492 else
493 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
494 }
495 } else {
496 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
497 }
498
499 /* MAIRs (stage-1 only) */
500 if (stage1) {
501 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
502 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
503 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
504 } else {
505 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
506 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
507 }
508 }
509 }
510
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)511 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
512 {
513 u32 reg;
514 bool stage1;
515 struct arm_smmu_cb *cb = &smmu->cbs[idx];
516 struct arm_smmu_cfg *cfg = cb->cfg;
517
518 /* Unassigned context banks only need disabling */
519 if (!cfg) {
520 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
521 return;
522 }
523
524 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
525
526 /* CBA2R */
527 if (smmu->version > ARM_SMMU_V1) {
528 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
529 reg = ARM_SMMU_CBA2R_VA64;
530 else
531 reg = 0;
532 /* 16-bit VMIDs live in CBA2R */
533 if (smmu->features & ARM_SMMU_FEAT_VMID16)
534 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
535
536 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
537 }
538
539 /* CBAR */
540 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
541 if (smmu->version < ARM_SMMU_V2)
542 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
543
544 /*
545 * Use the weakest shareability/memory types, so they are
546 * overridden by the ttbcr/pte.
547 */
548 if (stage1) {
549 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
550 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
551 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
552 ARM_SMMU_CBAR_S1_MEMATTR_WB);
553 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
554 /* 8-bit VMIDs live in CBAR */
555 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
556 }
557 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
558
559 /*
560 * TCR
561 * We must write this before the TTBRs, since it determines the
562 * access behaviour of some fields (in particular, ASID[15:8]).
563 */
564 if (stage1 && smmu->version > ARM_SMMU_V1)
565 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
566 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
567
568 /* TTBRs */
569 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
570 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
571 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
572 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
573 } else {
574 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
575 if (stage1)
576 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
577 cb->ttbr[1]);
578 }
579
580 /* MAIRs (stage-1 only) */
581 if (stage1) {
582 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
583 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
584 }
585
586 /* SCTLR */
587 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
588 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
589 if (stage1)
590 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
591 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
592 reg |= ARM_SMMU_SCTLR_E;
593
594 if (smmu->impl && smmu->impl->write_sctlr)
595 smmu->impl->write_sctlr(smmu, idx, reg);
596 else
597 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
598 }
599
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)600 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
601 struct arm_smmu_device *smmu,
602 struct device *dev, unsigned int start)
603 {
604 if (smmu->impl && smmu->impl->alloc_context_bank)
605 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
606
607 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
608 }
609
arm_smmu_init_domain_context(struct iommu_domain * domain,struct arm_smmu_device * smmu,struct device * dev)610 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
611 struct arm_smmu_device *smmu,
612 struct device *dev)
613 {
614 int irq, start, ret = 0;
615 unsigned long ias, oas;
616 struct io_pgtable_ops *pgtbl_ops;
617 struct io_pgtable_cfg pgtbl_cfg;
618 enum io_pgtable_fmt fmt;
619 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
620 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
621 irqreturn_t (*context_fault)(int irq, void *dev);
622
623 mutex_lock(&smmu_domain->init_mutex);
624 if (smmu_domain->smmu)
625 goto out_unlock;
626
627 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
628 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
629 smmu_domain->smmu = smmu;
630 goto out_unlock;
631 }
632
633 /*
634 * Mapping the requested stage onto what we support is surprisingly
635 * complicated, mainly because the spec allows S1+S2 SMMUs without
636 * support for nested translation. That means we end up with the
637 * following table:
638 *
639 * Requested Supported Actual
640 * S1 N S1
641 * S1 S1+S2 S1
642 * S1 S2 S2
643 * S1 S1 S1
644 * N N N
645 * N S1+S2 S2
646 * N S2 S2
647 * N S1 S1
648 *
649 * Note that you can't actually request stage-2 mappings.
650 */
651 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
652 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
653 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
654 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
655
656 /*
657 * Choosing a suitable context format is even more fiddly. Until we
658 * grow some way for the caller to express a preference, and/or move
659 * the decision into the io-pgtable code where it arguably belongs,
660 * just aim for the closest thing to the rest of the system, and hope
661 * that the hardware isn't esoteric enough that we can't assume AArch64
662 * support to be a superset of AArch32 support...
663 */
664 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
665 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
666 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
667 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
668 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
669 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
670 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
671 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
672 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
673 ARM_SMMU_FEAT_FMT_AARCH64_16K |
674 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
675 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
676
677 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
678 ret = -EINVAL;
679 goto out_unlock;
680 }
681
682 switch (smmu_domain->stage) {
683 case ARM_SMMU_DOMAIN_S1:
684 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
685 start = smmu->num_s2_context_banks;
686 ias = smmu->va_size;
687 oas = smmu->ipa_size;
688 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
689 fmt = ARM_64_LPAE_S1;
690 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
691 fmt = ARM_32_LPAE_S1;
692 ias = min(ias, 32UL);
693 oas = min(oas, 40UL);
694 } else {
695 fmt = ARM_V7S;
696 ias = min(ias, 32UL);
697 oas = min(oas, 32UL);
698 }
699 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
700 break;
701 case ARM_SMMU_DOMAIN_NESTED:
702 /*
703 * We will likely want to change this if/when KVM gets
704 * involved.
705 */
706 case ARM_SMMU_DOMAIN_S2:
707 cfg->cbar = CBAR_TYPE_S2_TRANS;
708 start = 0;
709 ias = smmu->ipa_size;
710 oas = smmu->pa_size;
711 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
712 fmt = ARM_64_LPAE_S2;
713 } else {
714 fmt = ARM_32_LPAE_S2;
715 ias = min(ias, 40UL);
716 oas = min(oas, 40UL);
717 }
718 if (smmu->version == ARM_SMMU_V2)
719 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
720 else
721 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
722 break;
723 default:
724 ret = -EINVAL;
725 goto out_unlock;
726 }
727
728 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
729 if (ret < 0) {
730 goto out_unlock;
731 }
732
733 smmu_domain->smmu = smmu;
734
735 cfg->cbndx = ret;
736 if (smmu->version < ARM_SMMU_V2) {
737 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
738 cfg->irptndx %= smmu->num_context_irqs;
739 } else {
740 cfg->irptndx = cfg->cbndx;
741 }
742
743 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
744 cfg->vmid = cfg->cbndx + 1;
745 else
746 cfg->asid = cfg->cbndx;
747
748 pgtbl_cfg = (struct io_pgtable_cfg) {
749 .pgsize_bitmap = smmu->pgsize_bitmap,
750 .ias = ias,
751 .oas = oas,
752 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
753 .tlb = smmu_domain->flush_ops,
754 .iommu_dev = smmu->dev,
755 };
756
757 if (smmu->impl && smmu->impl->init_context) {
758 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
759 if (ret)
760 goto out_clear_smmu;
761 }
762
763 if (smmu_domain->pgtbl_quirks)
764 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
765
766 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
767 if (!pgtbl_ops) {
768 ret = -ENOMEM;
769 goto out_clear_smmu;
770 }
771
772 /* Update the domain's page sizes to reflect the page table format */
773 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
774
775 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
776 domain->geometry.aperture_start = ~0UL << ias;
777 domain->geometry.aperture_end = ~0UL;
778 } else {
779 domain->geometry.aperture_end = (1UL << ias) - 1;
780 }
781
782 domain->geometry.force_aperture = true;
783
784 /* Initialise the context bank with our page table cfg */
785 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
786 arm_smmu_write_context_bank(smmu, cfg->cbndx);
787
788 /*
789 * Request context fault interrupt. Do this last to avoid the
790 * handler seeing a half-initialised domain state.
791 */
792 irq = smmu->irqs[cfg->irptndx];
793
794 if (smmu->impl && smmu->impl->context_fault)
795 context_fault = smmu->impl->context_fault;
796 else
797 context_fault = arm_smmu_context_fault;
798
799 ret = devm_request_irq(smmu->dev, irq, context_fault,
800 IRQF_SHARED, "arm-smmu-context-fault", domain);
801 if (ret < 0) {
802 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
803 cfg->irptndx, irq);
804 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
805 }
806
807 mutex_unlock(&smmu_domain->init_mutex);
808
809 /* Publish page table ops for map/unmap */
810 smmu_domain->pgtbl_ops = pgtbl_ops;
811 return 0;
812
813 out_clear_smmu:
814 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
815 smmu_domain->smmu = NULL;
816 out_unlock:
817 mutex_unlock(&smmu_domain->init_mutex);
818 return ret;
819 }
820
arm_smmu_destroy_domain_context(struct iommu_domain * domain)821 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
822 {
823 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
824 struct arm_smmu_device *smmu = smmu_domain->smmu;
825 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
826 int ret, irq;
827
828 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
829 return;
830
831 ret = arm_smmu_rpm_get(smmu);
832 if (ret < 0)
833 return;
834
835 /*
836 * Disable the context bank and free the page tables before freeing
837 * it.
838 */
839 smmu->cbs[cfg->cbndx].cfg = NULL;
840 arm_smmu_write_context_bank(smmu, cfg->cbndx);
841
842 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
843 irq = smmu->irqs[cfg->irptndx];
844 devm_free_irq(smmu->dev, irq, domain);
845 }
846
847 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
848 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
849
850 arm_smmu_rpm_put(smmu);
851 }
852
arm_smmu_domain_alloc(unsigned type)853 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
854 {
855 struct arm_smmu_domain *smmu_domain;
856
857 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
858 if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
859 return NULL;
860 }
861 /*
862 * Allocate the domain and initialise some of its data structures.
863 * We can't really do anything meaningful until we've added a
864 * master.
865 */
866 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
867 if (!smmu_domain)
868 return NULL;
869
870 mutex_init(&smmu_domain->init_mutex);
871 spin_lock_init(&smmu_domain->cb_lock);
872
873 return &smmu_domain->domain;
874 }
875
arm_smmu_domain_free(struct iommu_domain * domain)876 static void arm_smmu_domain_free(struct iommu_domain *domain)
877 {
878 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
879
880 /*
881 * Free the domain resources. We assume that all devices have
882 * already been detached.
883 */
884 arm_smmu_destroy_domain_context(domain);
885 kfree(smmu_domain);
886 }
887
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)888 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
889 {
890 struct arm_smmu_smr *smr = smmu->smrs + idx;
891 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
892 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
893
894 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
895 reg |= ARM_SMMU_SMR_VALID;
896 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
897 }
898
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)899 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
900 {
901 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
902 u32 reg;
903
904 if (smmu->impl && smmu->impl->write_s2cr) {
905 smmu->impl->write_s2cr(smmu, idx);
906 return;
907 }
908
909 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
910 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
911 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
912
913 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
914 smmu->smrs[idx].valid)
915 reg |= ARM_SMMU_S2CR_EXIDVALID;
916 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
917 }
918
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)919 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
920 {
921 arm_smmu_write_s2cr(smmu, idx);
922 if (smmu->smrs)
923 arm_smmu_write_smr(smmu, idx);
924 }
925
926 /*
927 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
928 * should be called after sCR0 is written.
929 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)930 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
931 {
932 u32 smr;
933 int i;
934
935 if (!smmu->smrs)
936 return;
937 /*
938 * If we've had to accommodate firmware memory regions, we may
939 * have live SMRs by now; tread carefully...
940 *
941 * Somewhat perversely, not having a free SMR for this test implies we
942 * can get away without it anyway, as we'll only be able to 'allocate'
943 * these SMRs for the ID/mask values we're already trusting to be OK.
944 */
945 for (i = 0; i < smmu->num_mapping_groups; i++)
946 if (!smmu->smrs[i].valid)
947 goto smr_ok;
948 return;
949 smr_ok:
950 /*
951 * SMR.ID bits may not be preserved if the corresponding MASK
952 * bits are set, so check each one separately. We can reject
953 * masters later if they try to claim IDs outside these masks.
954 */
955 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
956 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
957 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
958 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
959
960 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
961 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
962 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
963 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
964 }
965
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)966 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
967 {
968 struct arm_smmu_smr *smrs = smmu->smrs;
969 int i, free_idx = -ENOSPC;
970
971 /* Stream indexing is blissfully easy */
972 if (!smrs)
973 return id;
974
975 /* Validating SMRs is... less so */
976 for (i = 0; i < smmu->num_mapping_groups; ++i) {
977 if (!smrs[i].valid) {
978 /*
979 * Note the first free entry we come across, which
980 * we'll claim in the end if nothing else matches.
981 */
982 if (free_idx < 0)
983 free_idx = i;
984 continue;
985 }
986 /*
987 * If the new entry is _entirely_ matched by an existing entry,
988 * then reuse that, with the guarantee that there also cannot
989 * be any subsequent conflicting entries. In normal use we'd
990 * expect simply identical entries for this case, but there's
991 * no harm in accommodating the generalisation.
992 */
993 if ((mask & smrs[i].mask) == mask &&
994 !((id ^ smrs[i].id) & ~smrs[i].mask))
995 return i;
996 /*
997 * If the new entry has any other overlap with an existing one,
998 * though, then there always exists at least one stream ID
999 * which would cause a conflict, and we can't allow that risk.
1000 */
1001 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1002 return -EINVAL;
1003 }
1004
1005 return free_idx;
1006 }
1007
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1008 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1009 {
1010 if (--smmu->s2crs[idx].count)
1011 return false;
1012
1013 smmu->s2crs[idx] = s2cr_init_val;
1014 if (smmu->smrs)
1015 smmu->smrs[idx].valid = false;
1016
1017 return true;
1018 }
1019
arm_smmu_master_alloc_smes(struct device * dev)1020 static int arm_smmu_master_alloc_smes(struct device *dev)
1021 {
1022 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1023 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1024 struct arm_smmu_device *smmu = cfg->smmu;
1025 struct arm_smmu_smr *smrs = smmu->smrs;
1026 int i, idx, ret;
1027
1028 mutex_lock(&smmu->stream_map_mutex);
1029 /* Figure out a viable stream map entry allocation */
1030 for_each_cfg_sme(cfg, fwspec, i, idx) {
1031 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1032 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1033
1034 if (idx != INVALID_SMENDX) {
1035 ret = -EEXIST;
1036 goto out_err;
1037 }
1038
1039 ret = arm_smmu_find_sme(smmu, sid, mask);
1040 if (ret < 0)
1041 goto out_err;
1042
1043 idx = ret;
1044 if (smrs && smmu->s2crs[idx].count == 0) {
1045 smrs[idx].id = sid;
1046 smrs[idx].mask = mask;
1047 smrs[idx].valid = true;
1048 }
1049 smmu->s2crs[idx].count++;
1050 cfg->smendx[i] = (s16)idx;
1051 }
1052
1053 /* It worked! Now, poke the actual hardware */
1054 for_each_cfg_sme(cfg, fwspec, i, idx)
1055 arm_smmu_write_sme(smmu, idx);
1056
1057 mutex_unlock(&smmu->stream_map_mutex);
1058 return 0;
1059
1060 out_err:
1061 while (i--) {
1062 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1063 cfg->smendx[i] = INVALID_SMENDX;
1064 }
1065 mutex_unlock(&smmu->stream_map_mutex);
1066 return ret;
1067 }
1068
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1069 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1070 struct iommu_fwspec *fwspec)
1071 {
1072 struct arm_smmu_device *smmu = cfg->smmu;
1073 int i, idx;
1074
1075 mutex_lock(&smmu->stream_map_mutex);
1076 for_each_cfg_sme(cfg, fwspec, i, idx) {
1077 if (arm_smmu_free_sme(smmu, idx))
1078 arm_smmu_write_sme(smmu, idx);
1079 cfg->smendx[i] = INVALID_SMENDX;
1080 }
1081 mutex_unlock(&smmu->stream_map_mutex);
1082 }
1083
arm_smmu_domain_add_master(struct arm_smmu_domain * smmu_domain,struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1084 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1085 struct arm_smmu_master_cfg *cfg,
1086 struct iommu_fwspec *fwspec)
1087 {
1088 struct arm_smmu_device *smmu = smmu_domain->smmu;
1089 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1090 u8 cbndx = smmu_domain->cfg.cbndx;
1091 enum arm_smmu_s2cr_type type;
1092 int i, idx;
1093
1094 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1095 type = S2CR_TYPE_BYPASS;
1096 else
1097 type = S2CR_TYPE_TRANS;
1098
1099 for_each_cfg_sme(cfg, fwspec, i, idx) {
1100 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1101 continue;
1102
1103 s2cr[idx].type = type;
1104 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1105 s2cr[idx].cbndx = cbndx;
1106 arm_smmu_write_s2cr(smmu, idx);
1107 }
1108 return 0;
1109 }
1110
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1111 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1112 {
1113 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1114 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1115 struct arm_smmu_master_cfg *cfg;
1116 struct arm_smmu_device *smmu;
1117 int ret;
1118
1119 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1120 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1121 return -ENXIO;
1122 }
1123
1124 /*
1125 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1126 * domains between of_xlate() and probe_device() - we have no way to cope
1127 * with that, so until ARM gets converted to rely on groups and default
1128 * domains, just say no (but more politely than by dereferencing NULL).
1129 * This should be at least a WARN_ON once that's sorted.
1130 */
1131 cfg = dev_iommu_priv_get(dev);
1132 if (!cfg)
1133 return -ENODEV;
1134
1135 smmu = cfg->smmu;
1136
1137 ret = arm_smmu_rpm_get(smmu);
1138 if (ret < 0)
1139 return ret;
1140
1141 /* Ensure that the domain is finalised */
1142 ret = arm_smmu_init_domain_context(domain, smmu, dev);
1143 if (ret < 0)
1144 goto rpm_put;
1145
1146 /*
1147 * Sanity check the domain. We don't support domains across
1148 * different SMMUs.
1149 */
1150 if (smmu_domain->smmu != smmu) {
1151 ret = -EINVAL;
1152 goto rpm_put;
1153 }
1154
1155 /* Looks ok, so add the device to the domain */
1156 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1157
1158 /*
1159 * Setup an autosuspend delay to avoid bouncing runpm state.
1160 * Otherwise, if a driver for a suspended consumer device
1161 * unmaps buffers, it will runpm resume/suspend for each one.
1162 *
1163 * For example, when used by a GPU device, when an application
1164 * or game exits, it can trigger unmapping 100s or 1000s of
1165 * buffers. With a runpm cycle for each buffer, that adds up
1166 * to 5-10sec worth of reprogramming the context bank, while
1167 * the system appears to be locked up to the user.
1168 */
1169 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1170 pm_runtime_use_autosuspend(smmu->dev);
1171
1172 rpm_put:
1173 arm_smmu_rpm_put(smmu);
1174 return ret;
1175 }
1176
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1177 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1178 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1179 int prot, gfp_t gfp, size_t *mapped)
1180 {
1181 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1182 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1183 int ret;
1184
1185 if (!ops)
1186 return -ENODEV;
1187
1188 arm_smmu_rpm_get(smmu);
1189 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1190 arm_smmu_rpm_put(smmu);
1191
1192 return ret;
1193 }
1194
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1195 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1196 size_t pgsize, size_t pgcount,
1197 struct iommu_iotlb_gather *iotlb_gather)
1198 {
1199 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1200 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1201 size_t ret;
1202
1203 if (!ops)
1204 return 0;
1205
1206 arm_smmu_rpm_get(smmu);
1207 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1208 arm_smmu_rpm_put(smmu);
1209
1210 return ret;
1211 }
1212
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1213 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1214 {
1215 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216 struct arm_smmu_device *smmu = smmu_domain->smmu;
1217
1218 if (smmu_domain->flush_ops) {
1219 arm_smmu_rpm_get(smmu);
1220 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1221 arm_smmu_rpm_put(smmu);
1222 }
1223 }
1224
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1225 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1226 struct iommu_iotlb_gather *gather)
1227 {
1228 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1229 struct arm_smmu_device *smmu = smmu_domain->smmu;
1230
1231 if (!smmu)
1232 return;
1233
1234 arm_smmu_rpm_get(smmu);
1235 if (smmu->version == ARM_SMMU_V2 ||
1236 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1237 arm_smmu_tlb_sync_context(smmu_domain);
1238 else
1239 arm_smmu_tlb_sync_global(smmu);
1240 arm_smmu_rpm_put(smmu);
1241 }
1242
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1243 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1244 dma_addr_t iova)
1245 {
1246 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1247 struct arm_smmu_device *smmu = smmu_domain->smmu;
1248 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1249 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1250 struct device *dev = smmu->dev;
1251 void __iomem *reg;
1252 u32 tmp;
1253 u64 phys;
1254 unsigned long va, flags;
1255 int ret, idx = cfg->cbndx;
1256 phys_addr_t addr = 0;
1257
1258 ret = arm_smmu_rpm_get(smmu);
1259 if (ret < 0)
1260 return 0;
1261
1262 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1263 va = iova & ~0xfffUL;
1264 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1265 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1266 else
1267 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1268
1269 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1270 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1271 5, 50)) {
1272 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1273 dev_err(dev,
1274 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1275 &iova);
1276 arm_smmu_rpm_put(smmu);
1277 return ops->iova_to_phys(ops, iova);
1278 }
1279
1280 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1281 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282 if (phys & ARM_SMMU_CB_PAR_F) {
1283 dev_err(dev, "translation fault!\n");
1284 dev_err(dev, "PAR = 0x%llx\n", phys);
1285 goto out;
1286 }
1287
1288 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1289 out:
1290 arm_smmu_rpm_put(smmu);
1291
1292 return addr;
1293 }
1294
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1295 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1296 dma_addr_t iova)
1297 {
1298 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1299 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1300
1301 if (!ops)
1302 return 0;
1303
1304 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1305 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1306 return arm_smmu_iova_to_phys_hard(domain, iova);
1307
1308 return ops->iova_to_phys(ops, iova);
1309 }
1310
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1311 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1312 {
1313 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1314
1315 switch (cap) {
1316 case IOMMU_CAP_CACHE_COHERENCY:
1317 /*
1318 * It's overwhelmingly the case in practice that when the pagetable
1319 * walk interface is connected to a coherent interconnect, all the
1320 * translation interfaces are too. Furthermore if the device is
1321 * natively coherent, then its translation interface must also be.
1322 */
1323 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1324 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1325 case IOMMU_CAP_NOEXEC:
1326 case IOMMU_CAP_DEFERRED_FLUSH:
1327 return true;
1328 default:
1329 return false;
1330 }
1331 }
1332
1333 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1334 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1335 {
1336 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1337 fwnode);
1338 put_device(dev);
1339 return dev ? dev_get_drvdata(dev) : NULL;
1340 }
1341
arm_smmu_probe_device(struct device * dev)1342 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1343 {
1344 struct arm_smmu_device *smmu = NULL;
1345 struct arm_smmu_master_cfg *cfg;
1346 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1347 int i, ret;
1348
1349 if (using_legacy_binding) {
1350 ret = arm_smmu_register_legacy_master(dev, &smmu);
1351
1352 /*
1353 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1354 * will allocate/initialise a new one. Thus we need to update fwspec for
1355 * later use.
1356 */
1357 fwspec = dev_iommu_fwspec_get(dev);
1358 if (ret)
1359 goto out_free;
1360 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1361 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1362 } else {
1363 return ERR_PTR(-ENODEV);
1364 }
1365
1366 ret = -EINVAL;
1367 for (i = 0; i < fwspec->num_ids; i++) {
1368 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1369 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1370
1371 if (sid & ~smmu->streamid_mask) {
1372 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1373 sid, smmu->streamid_mask);
1374 goto out_free;
1375 }
1376 if (mask & ~smmu->smr_mask_mask) {
1377 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1378 mask, smmu->smr_mask_mask);
1379 goto out_free;
1380 }
1381 }
1382
1383 ret = -ENOMEM;
1384 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1385 GFP_KERNEL);
1386 if (!cfg)
1387 goto out_free;
1388
1389 cfg->smmu = smmu;
1390 dev_iommu_priv_set(dev, cfg);
1391 while (i--)
1392 cfg->smendx[i] = INVALID_SMENDX;
1393
1394 ret = arm_smmu_rpm_get(smmu);
1395 if (ret < 0)
1396 goto out_cfg_free;
1397
1398 ret = arm_smmu_master_alloc_smes(dev);
1399 arm_smmu_rpm_put(smmu);
1400
1401 if (ret)
1402 goto out_cfg_free;
1403
1404 device_link_add(dev, smmu->dev,
1405 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1406
1407 return &smmu->iommu;
1408
1409 out_cfg_free:
1410 kfree(cfg);
1411 out_free:
1412 iommu_fwspec_free(dev);
1413 return ERR_PTR(ret);
1414 }
1415
arm_smmu_release_device(struct device * dev)1416 static void arm_smmu_release_device(struct device *dev)
1417 {
1418 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1419 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1420 int ret;
1421
1422 ret = arm_smmu_rpm_get(cfg->smmu);
1423 if (ret < 0)
1424 return;
1425
1426 arm_smmu_master_free_smes(cfg, fwspec);
1427
1428 arm_smmu_rpm_put(cfg->smmu);
1429
1430 dev_iommu_priv_set(dev, NULL);
1431 kfree(cfg);
1432 }
1433
arm_smmu_probe_finalize(struct device * dev)1434 static void arm_smmu_probe_finalize(struct device *dev)
1435 {
1436 struct arm_smmu_master_cfg *cfg;
1437 struct arm_smmu_device *smmu;
1438
1439 cfg = dev_iommu_priv_get(dev);
1440 smmu = cfg->smmu;
1441
1442 if (smmu->impl && smmu->impl->probe_finalize)
1443 smmu->impl->probe_finalize(smmu, dev);
1444 }
1445
arm_smmu_device_group(struct device * dev)1446 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1447 {
1448 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1449 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1450 struct arm_smmu_device *smmu = cfg->smmu;
1451 struct iommu_group *group = NULL;
1452 int i, idx;
1453
1454 mutex_lock(&smmu->stream_map_mutex);
1455 for_each_cfg_sme(cfg, fwspec, i, idx) {
1456 if (group && smmu->s2crs[idx].group &&
1457 group != smmu->s2crs[idx].group) {
1458 mutex_unlock(&smmu->stream_map_mutex);
1459 return ERR_PTR(-EINVAL);
1460 }
1461
1462 group = smmu->s2crs[idx].group;
1463 }
1464
1465 if (group) {
1466 mutex_unlock(&smmu->stream_map_mutex);
1467 return iommu_group_ref_get(group);
1468 }
1469
1470 if (dev_is_pci(dev))
1471 group = pci_device_group(dev);
1472 else if (dev_is_fsl_mc(dev))
1473 group = fsl_mc_device_group(dev);
1474 else
1475 group = generic_device_group(dev);
1476
1477 /* Remember group for faster lookups */
1478 if (!IS_ERR(group))
1479 for_each_cfg_sme(cfg, fwspec, i, idx)
1480 smmu->s2crs[idx].group = group;
1481
1482 mutex_unlock(&smmu->stream_map_mutex);
1483 return group;
1484 }
1485
arm_smmu_enable_nesting(struct iommu_domain * domain)1486 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1487 {
1488 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1489 int ret = 0;
1490
1491 mutex_lock(&smmu_domain->init_mutex);
1492 if (smmu_domain->smmu)
1493 ret = -EPERM;
1494 else
1495 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1496 mutex_unlock(&smmu_domain->init_mutex);
1497
1498 return ret;
1499 }
1500
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1501 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1502 unsigned long quirks)
1503 {
1504 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1505 int ret = 0;
1506
1507 mutex_lock(&smmu_domain->init_mutex);
1508 if (smmu_domain->smmu)
1509 ret = -EPERM;
1510 else
1511 smmu_domain->pgtbl_quirks = quirks;
1512 mutex_unlock(&smmu_domain->init_mutex);
1513
1514 return ret;
1515 }
1516
arm_smmu_of_xlate(struct device * dev,struct of_phandle_args * args)1517 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1518 {
1519 u32 mask, fwid = 0;
1520
1521 if (args->args_count > 0)
1522 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1523
1524 if (args->args_count > 1)
1525 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1526 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1527 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1528
1529 return iommu_fwspec_add_ids(dev, &fwid, 1);
1530 }
1531
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1532 static void arm_smmu_get_resv_regions(struct device *dev,
1533 struct list_head *head)
1534 {
1535 struct iommu_resv_region *region;
1536 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1537
1538 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1539 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1540 if (!region)
1541 return;
1542
1543 list_add_tail(®ion->list, head);
1544
1545 iommu_dma_get_resv_regions(dev, head);
1546 }
1547
arm_smmu_def_domain_type(struct device * dev)1548 static int arm_smmu_def_domain_type(struct device *dev)
1549 {
1550 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1551 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1552
1553 if (using_legacy_binding)
1554 return IOMMU_DOMAIN_IDENTITY;
1555
1556 if (impl && impl->def_domain_type)
1557 return impl->def_domain_type(dev);
1558
1559 return 0;
1560 }
1561
1562 static struct iommu_ops arm_smmu_ops = {
1563 .capable = arm_smmu_capable,
1564 .domain_alloc = arm_smmu_domain_alloc,
1565 .probe_device = arm_smmu_probe_device,
1566 .release_device = arm_smmu_release_device,
1567 .probe_finalize = arm_smmu_probe_finalize,
1568 .device_group = arm_smmu_device_group,
1569 .of_xlate = arm_smmu_of_xlate,
1570 .get_resv_regions = arm_smmu_get_resv_regions,
1571 .def_domain_type = arm_smmu_def_domain_type,
1572 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1573 .owner = THIS_MODULE,
1574 .default_domain_ops = &(const struct iommu_domain_ops) {
1575 .attach_dev = arm_smmu_attach_dev,
1576 .map_pages = arm_smmu_map_pages,
1577 .unmap_pages = arm_smmu_unmap_pages,
1578 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1579 .iotlb_sync = arm_smmu_iotlb_sync,
1580 .iova_to_phys = arm_smmu_iova_to_phys,
1581 .enable_nesting = arm_smmu_enable_nesting,
1582 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1583 .free = arm_smmu_domain_free,
1584 }
1585 };
1586
arm_smmu_device_reset(struct arm_smmu_device * smmu)1587 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1588 {
1589 int i;
1590 u32 reg;
1591
1592 /* clear global FSR */
1593 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1594 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1595
1596 /*
1597 * Reset stream mapping groups: Initial values mark all SMRn as
1598 * invalid and all S2CRn as bypass unless overridden.
1599 */
1600 for (i = 0; i < smmu->num_mapping_groups; ++i)
1601 arm_smmu_write_sme(smmu, i);
1602
1603 /* Make sure all context banks are disabled and clear CB_FSR */
1604 for (i = 0; i < smmu->num_context_banks; ++i) {
1605 arm_smmu_write_context_bank(smmu, i);
1606 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1607 }
1608
1609 /* Invalidate the TLB, just in case */
1610 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1611 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1612
1613 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1614
1615 /* Enable fault reporting */
1616 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1617 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1618
1619 /* Disable TLB broadcasting. */
1620 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1621
1622 /* Enable client access, handling unmatched streams as appropriate */
1623 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1624 if (disable_bypass)
1625 reg |= ARM_SMMU_sCR0_USFCFG;
1626 else
1627 reg &= ~ARM_SMMU_sCR0_USFCFG;
1628
1629 /* Disable forced broadcasting */
1630 reg &= ~ARM_SMMU_sCR0_FB;
1631
1632 /* Don't upgrade barriers */
1633 reg &= ~(ARM_SMMU_sCR0_BSU);
1634
1635 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1636 reg |= ARM_SMMU_sCR0_VMID16EN;
1637
1638 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1639 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1640
1641 if (smmu->impl && smmu->impl->reset)
1642 smmu->impl->reset(smmu);
1643
1644 /* Push the button */
1645 arm_smmu_tlb_sync_global(smmu);
1646 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1647 }
1648
arm_smmu_id_size_to_bits(int size)1649 static int arm_smmu_id_size_to_bits(int size)
1650 {
1651 switch (size) {
1652 case 0:
1653 return 32;
1654 case 1:
1655 return 36;
1656 case 2:
1657 return 40;
1658 case 3:
1659 return 42;
1660 case 4:
1661 return 44;
1662 case 5:
1663 default:
1664 return 48;
1665 }
1666 }
1667
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1668 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1669 {
1670 unsigned int size;
1671 u32 id;
1672 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1673 int i, ret;
1674
1675 dev_notice(smmu->dev, "probing hardware configuration...\n");
1676 dev_notice(smmu->dev, "SMMUv%d with:\n",
1677 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1678
1679 /* ID0 */
1680 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1681
1682 /* Restrict available stages based on module parameter */
1683 if (force_stage == 1)
1684 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1685 else if (force_stage == 2)
1686 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1687
1688 if (id & ARM_SMMU_ID0_S1TS) {
1689 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1690 dev_notice(smmu->dev, "\tstage 1 translation\n");
1691 }
1692
1693 if (id & ARM_SMMU_ID0_S2TS) {
1694 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1695 dev_notice(smmu->dev, "\tstage 2 translation\n");
1696 }
1697
1698 if (id & ARM_SMMU_ID0_NTS) {
1699 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1700 dev_notice(smmu->dev, "\tnested translation\n");
1701 }
1702
1703 if (!(smmu->features &
1704 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1705 dev_err(smmu->dev, "\tno translation support!\n");
1706 return -ENODEV;
1707 }
1708
1709 if ((id & ARM_SMMU_ID0_S1TS) &&
1710 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1711 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1712 dev_notice(smmu->dev, "\taddress translation ops\n");
1713 }
1714
1715 /*
1716 * In order for DMA API calls to work properly, we must defer to what
1717 * the FW says about coherency, regardless of what the hardware claims.
1718 * Fortunately, this also opens up a workaround for systems where the
1719 * ID register value has ended up configured incorrectly.
1720 */
1721 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1722 if (cttw_fw || cttw_reg)
1723 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1724 cttw_fw ? "" : "non-");
1725 if (cttw_fw != cttw_reg)
1726 dev_notice(smmu->dev,
1727 "\t(IDR0.CTTW overridden by FW configuration)\n");
1728
1729 /* Max. number of entries we have for stream matching/indexing */
1730 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1731 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1732 size = 1 << 16;
1733 } else {
1734 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1735 }
1736 smmu->streamid_mask = size - 1;
1737 if (id & ARM_SMMU_ID0_SMS) {
1738 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1739 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1740 if (size == 0) {
1741 dev_err(smmu->dev,
1742 "stream-matching supported, but no SMRs present!\n");
1743 return -ENODEV;
1744 }
1745
1746 /* Zero-initialised to mark as invalid */
1747 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1748 GFP_KERNEL);
1749 if (!smmu->smrs)
1750 return -ENOMEM;
1751
1752 dev_notice(smmu->dev,
1753 "\tstream matching with %u register groups", size);
1754 }
1755 /* s2cr->type == 0 means translation, so initialise explicitly */
1756 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1757 GFP_KERNEL);
1758 if (!smmu->s2crs)
1759 return -ENOMEM;
1760 for (i = 0; i < size; i++)
1761 smmu->s2crs[i] = s2cr_init_val;
1762
1763 smmu->num_mapping_groups = size;
1764 mutex_init(&smmu->stream_map_mutex);
1765 spin_lock_init(&smmu->global_sync_lock);
1766
1767 if (smmu->version < ARM_SMMU_V2 ||
1768 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1769 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1770 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1771 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1772 }
1773
1774 /* ID1 */
1775 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1776 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1777
1778 /* Check for size mismatch of SMMU address space from mapped region */
1779 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1780 if (smmu->numpage != 2 * size << smmu->pgshift)
1781 dev_warn(smmu->dev,
1782 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1783 2 * size << smmu->pgshift, smmu->numpage);
1784 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1785 smmu->numpage = size;
1786
1787 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1788 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1789 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1790 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1791 return -ENODEV;
1792 }
1793 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1794 smmu->num_context_banks, smmu->num_s2_context_banks);
1795 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1796 sizeof(*smmu->cbs), GFP_KERNEL);
1797 if (!smmu->cbs)
1798 return -ENOMEM;
1799
1800 /* ID2 */
1801 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1802 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1803 smmu->ipa_size = size;
1804
1805 /* The output mask is also applied for bypass */
1806 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1807 smmu->pa_size = size;
1808
1809 if (id & ARM_SMMU_ID2_VMID16)
1810 smmu->features |= ARM_SMMU_FEAT_VMID16;
1811
1812 /*
1813 * What the page table walker can address actually depends on which
1814 * descriptor format is in use, but since a) we don't know that yet,
1815 * and b) it can vary per context bank, this will have to do...
1816 */
1817 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1818 dev_warn(smmu->dev,
1819 "failed to set DMA mask for table walker\n");
1820
1821 if (smmu->version < ARM_SMMU_V2) {
1822 smmu->va_size = smmu->ipa_size;
1823 if (smmu->version == ARM_SMMU_V1_64K)
1824 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1825 } else {
1826 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1827 smmu->va_size = arm_smmu_id_size_to_bits(size);
1828 if (id & ARM_SMMU_ID2_PTFS_4K)
1829 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1830 if (id & ARM_SMMU_ID2_PTFS_16K)
1831 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1832 if (id & ARM_SMMU_ID2_PTFS_64K)
1833 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1834 }
1835
1836 if (smmu->impl && smmu->impl->cfg_probe) {
1837 ret = smmu->impl->cfg_probe(smmu);
1838 if (ret)
1839 return ret;
1840 }
1841
1842 /* Now we've corralled the various formats, what'll it do? */
1843 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1844 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1845 if (smmu->features &
1846 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1847 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1848 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1849 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1850 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1851 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1852
1853 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1854 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1855 else
1856 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1857 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1858 smmu->pgsize_bitmap);
1859
1860
1861 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1862 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1863 smmu->va_size, smmu->ipa_size);
1864
1865 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1866 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1867 smmu->ipa_size, smmu->pa_size);
1868
1869 return 0;
1870 }
1871
1872 struct arm_smmu_match_data {
1873 enum arm_smmu_arch_version version;
1874 enum arm_smmu_implementation model;
1875 };
1876
1877 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1878 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1879
1880 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1881 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1882 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1883 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1884 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1885 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1886
1887 static const struct of_device_id arm_smmu_of_match[] = {
1888 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1889 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1890 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1891 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1892 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1893 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1894 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1895 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1896 { },
1897 };
1898 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1899
1900 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1901 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1902 {
1903 int ret = 0;
1904
1905 switch (model) {
1906 case ACPI_IORT_SMMU_V1:
1907 case ACPI_IORT_SMMU_CORELINK_MMU400:
1908 smmu->version = ARM_SMMU_V1;
1909 smmu->model = GENERIC_SMMU;
1910 break;
1911 case ACPI_IORT_SMMU_CORELINK_MMU401:
1912 smmu->version = ARM_SMMU_V1_64K;
1913 smmu->model = GENERIC_SMMU;
1914 break;
1915 case ACPI_IORT_SMMU_V2:
1916 smmu->version = ARM_SMMU_V2;
1917 smmu->model = GENERIC_SMMU;
1918 break;
1919 case ACPI_IORT_SMMU_CORELINK_MMU500:
1920 smmu->version = ARM_SMMU_V2;
1921 smmu->model = ARM_MMU500;
1922 break;
1923 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1924 smmu->version = ARM_SMMU_V2;
1925 smmu->model = CAVIUM_SMMUV2;
1926 break;
1927 default:
1928 ret = -ENODEV;
1929 }
1930
1931 return ret;
1932 }
1933
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1934 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1935 u32 *global_irqs, u32 *pmu_irqs)
1936 {
1937 struct device *dev = smmu->dev;
1938 struct acpi_iort_node *node =
1939 *(struct acpi_iort_node **)dev_get_platdata(dev);
1940 struct acpi_iort_smmu *iort_smmu;
1941 int ret;
1942
1943 /* Retrieve SMMU1/2 specific data */
1944 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1945
1946 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1947 if (ret < 0)
1948 return ret;
1949
1950 /* Ignore the configuration access interrupt */
1951 *global_irqs = 1;
1952 *pmu_irqs = 0;
1953
1954 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1955 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1956
1957 return 0;
1958 }
1959 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1960 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1961 u32 *global_irqs, u32 *pmu_irqs)
1962 {
1963 return -ENODEV;
1964 }
1965 #endif
1966
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1967 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
1968 u32 *global_irqs, u32 *pmu_irqs)
1969 {
1970 const struct arm_smmu_match_data *data;
1971 struct device *dev = smmu->dev;
1972 bool legacy_binding;
1973
1974 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
1975 return dev_err_probe(dev, -ENODEV,
1976 "missing #global-interrupts property\n");
1977 *pmu_irqs = 0;
1978
1979 data = of_device_get_match_data(dev);
1980 smmu->version = data->version;
1981 smmu->model = data->model;
1982
1983 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1984 if (legacy_binding && !using_generic_binding) {
1985 if (!using_legacy_binding) {
1986 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
1987 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
1988 }
1989 using_legacy_binding = true;
1990 } else if (!legacy_binding && !using_legacy_binding) {
1991 using_generic_binding = true;
1992 } else {
1993 dev_err(dev, "not probing due to mismatched DT properties\n");
1994 return -ENODEV;
1995 }
1996
1997 if (of_dma_is_coherent(dev->of_node))
1998 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1999
2000 return 0;
2001 }
2002
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2003 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2004 {
2005 struct list_head rmr_list;
2006 struct iommu_resv_region *e;
2007 int idx, cnt = 0;
2008 u32 reg;
2009
2010 INIT_LIST_HEAD(&rmr_list);
2011 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2012
2013 /*
2014 * Rather than trying to look at existing mappings that
2015 * are setup by the firmware and then invalidate the ones
2016 * that do no have matching RMR entries, just disable the
2017 * SMMU until it gets enabled again in the reset routine.
2018 */
2019 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2020 reg |= ARM_SMMU_sCR0_CLIENTPD;
2021 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2022
2023 list_for_each_entry(e, &rmr_list, list) {
2024 struct iommu_iort_rmr_data *rmr;
2025 int i;
2026
2027 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2028 for (i = 0; i < rmr->num_sids; i++) {
2029 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2030 if (idx < 0)
2031 continue;
2032
2033 if (smmu->s2crs[idx].count == 0) {
2034 smmu->smrs[idx].id = rmr->sids[i];
2035 smmu->smrs[idx].mask = 0;
2036 smmu->smrs[idx].valid = true;
2037 }
2038 smmu->s2crs[idx].count++;
2039 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2040 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2041
2042 cnt++;
2043 }
2044 }
2045
2046 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2047 cnt == 1 ? "" : "s");
2048 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2049 }
2050
arm_smmu_device_probe(struct platform_device * pdev)2051 static int arm_smmu_device_probe(struct platform_device *pdev)
2052 {
2053 struct resource *res;
2054 struct arm_smmu_device *smmu;
2055 struct device *dev = &pdev->dev;
2056 int num_irqs, i, err;
2057 u32 global_irqs, pmu_irqs;
2058 irqreturn_t (*global_fault)(int irq, void *dev);
2059
2060 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2061 if (!smmu) {
2062 dev_err(dev, "failed to allocate arm_smmu_device\n");
2063 return -ENOMEM;
2064 }
2065 smmu->dev = dev;
2066
2067 if (dev->of_node)
2068 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2069 else
2070 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2071 if (err)
2072 return err;
2073
2074 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2075 if (IS_ERR(smmu->base))
2076 return PTR_ERR(smmu->base);
2077 smmu->ioaddr = res->start;
2078
2079 /*
2080 * The resource size should effectively match the value of SMMU_TOP;
2081 * stash that temporarily until we know PAGESIZE to validate it with.
2082 */
2083 smmu->numpage = resource_size(res);
2084
2085 smmu = arm_smmu_impl_init(smmu);
2086 if (IS_ERR(smmu))
2087 return PTR_ERR(smmu);
2088
2089 num_irqs = platform_irq_count(pdev);
2090
2091 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2092 if (smmu->num_context_irqs <= 0)
2093 return dev_err_probe(dev, -ENODEV,
2094 "found %d interrupts but expected at least %d\n",
2095 num_irqs, global_irqs + pmu_irqs + 1);
2096
2097 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2098 sizeof(*smmu->irqs), GFP_KERNEL);
2099 if (!smmu->irqs)
2100 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2101 smmu->num_context_irqs);
2102
2103 for (i = 0; i < smmu->num_context_irqs; i++) {
2104 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2105
2106 if (irq < 0)
2107 return irq;
2108 smmu->irqs[i] = irq;
2109 }
2110
2111 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2112 if (err < 0) {
2113 dev_err(dev, "failed to get clocks %d\n", err);
2114 return err;
2115 }
2116 smmu->num_clks = err;
2117
2118 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2119 if (err)
2120 return err;
2121
2122 err = arm_smmu_device_cfg_probe(smmu);
2123 if (err)
2124 return err;
2125
2126 if (smmu->version == ARM_SMMU_V2) {
2127 if (smmu->num_context_banks > smmu->num_context_irqs) {
2128 dev_err(dev,
2129 "found only %d context irq(s) but %d required\n",
2130 smmu->num_context_irqs, smmu->num_context_banks);
2131 return -ENODEV;
2132 }
2133
2134 /* Ignore superfluous interrupts */
2135 smmu->num_context_irqs = smmu->num_context_banks;
2136 }
2137
2138 if (smmu->impl && smmu->impl->global_fault)
2139 global_fault = smmu->impl->global_fault;
2140 else
2141 global_fault = arm_smmu_global_fault;
2142
2143 for (i = 0; i < global_irqs; i++) {
2144 int irq = platform_get_irq(pdev, i);
2145
2146 if (irq < 0)
2147 return irq;
2148
2149 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2150 "arm-smmu global fault", smmu);
2151 if (err)
2152 return dev_err_probe(dev, err,
2153 "failed to request global IRQ %d (%u)\n",
2154 i, irq);
2155 }
2156
2157 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2158 "smmu.%pa", &smmu->ioaddr);
2159 if (err) {
2160 dev_err(dev, "Failed to register iommu in sysfs\n");
2161 return err;
2162 }
2163
2164 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
2165 if (err) {
2166 dev_err(dev, "Failed to register iommu\n");
2167 iommu_device_sysfs_remove(&smmu->iommu);
2168 return err;
2169 }
2170
2171 platform_set_drvdata(pdev, smmu);
2172
2173 /* Check for RMRs and install bypass SMRs if any */
2174 arm_smmu_rmr_install_bypass_smr(smmu);
2175
2176 arm_smmu_device_reset(smmu);
2177 arm_smmu_test_smr_masks(smmu);
2178
2179 /*
2180 * We want to avoid touching dev->power.lock in fastpaths unless
2181 * it's really going to do something useful - pm_runtime_enabled()
2182 * can serve as an ideal proxy for that decision. So, conditionally
2183 * enable pm_runtime.
2184 */
2185 if (dev->pm_domain) {
2186 pm_runtime_set_active(dev);
2187 pm_runtime_enable(dev);
2188 }
2189
2190 return 0;
2191 }
2192
arm_smmu_device_shutdown(struct platform_device * pdev)2193 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2194 {
2195 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2196
2197 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2198 dev_notice(&pdev->dev, "disabling translation\n");
2199
2200 arm_smmu_rpm_get(smmu);
2201 /* Turn the thing off */
2202 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2203 arm_smmu_rpm_put(smmu);
2204
2205 if (pm_runtime_enabled(smmu->dev))
2206 pm_runtime_force_suspend(smmu->dev);
2207 else
2208 clk_bulk_disable(smmu->num_clks, smmu->clks);
2209
2210 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2211 }
2212
arm_smmu_device_remove(struct platform_device * pdev)2213 static void arm_smmu_device_remove(struct platform_device *pdev)
2214 {
2215 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2216
2217 iommu_device_unregister(&smmu->iommu);
2218 iommu_device_sysfs_remove(&smmu->iommu);
2219
2220 arm_smmu_device_shutdown(pdev);
2221 }
2222
arm_smmu_runtime_resume(struct device * dev)2223 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2224 {
2225 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2226 int ret;
2227
2228 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2229 if (ret)
2230 return ret;
2231
2232 arm_smmu_device_reset(smmu);
2233
2234 return 0;
2235 }
2236
arm_smmu_runtime_suspend(struct device * dev)2237 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2238 {
2239 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2240
2241 clk_bulk_disable(smmu->num_clks, smmu->clks);
2242
2243 return 0;
2244 }
2245
arm_smmu_pm_resume(struct device * dev)2246 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2247 {
2248 int ret;
2249 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2250
2251 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2252 if (ret)
2253 return ret;
2254
2255 if (pm_runtime_suspended(dev))
2256 return 0;
2257
2258 ret = arm_smmu_runtime_resume(dev);
2259 if (ret)
2260 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2261
2262 return ret;
2263 }
2264
arm_smmu_pm_suspend(struct device * dev)2265 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2266 {
2267 int ret = 0;
2268 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2269
2270 if (pm_runtime_suspended(dev))
2271 goto clk_unprepare;
2272
2273 ret = arm_smmu_runtime_suspend(dev);
2274 if (ret)
2275 return ret;
2276
2277 clk_unprepare:
2278 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2279 return ret;
2280 }
2281
2282 static const struct dev_pm_ops arm_smmu_pm_ops = {
2283 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2284 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2285 arm_smmu_runtime_resume, NULL)
2286 };
2287
2288 static struct platform_driver arm_smmu_driver = {
2289 .driver = {
2290 .name = "arm-smmu",
2291 .of_match_table = arm_smmu_of_match,
2292 .pm = &arm_smmu_pm_ops,
2293 .suppress_bind_attrs = true,
2294 },
2295 .probe = arm_smmu_device_probe,
2296 .remove_new = arm_smmu_device_remove,
2297 .shutdown = arm_smmu_device_shutdown,
2298 };
2299 module_platform_driver(arm_smmu_driver);
2300
2301 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2302 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2303 MODULE_ALIAS("platform:arm-smmu");
2304 MODULE_LICENSE("GPL v2");
2305
