1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2019, The Linux Foundation. All rights reserved.
4 */
5
6 #include <linux/acpi.h>
7 #include <linux/adreno-smmu-priv.h>
8 #include <linux/of_device.h>
9 #include <linux/qcom_scm.h>
10
11 #include "arm-smmu.h"
12
13 struct qcom_smmu {
14 struct arm_smmu_device smmu;
15 bool bypass_quirk;
16 u8 bypass_cbndx;
17 u32 stall_enabled;
18 };
19
to_qcom_smmu(struct arm_smmu_device * smmu)20 static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu)
21 {
22 return container_of(smmu, struct qcom_smmu, smmu);
23 }
24
qcom_adreno_smmu_write_sctlr(struct arm_smmu_device * smmu,int idx,u32 reg)25 static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx,
26 u32 reg)
27 {
28 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
29
30 /*
31 * On the GPU device we want to process subsequent transactions after a
32 * fault to keep the GPU from hanging
33 */
34 reg |= ARM_SMMU_SCTLR_HUPCF;
35
36 if (qsmmu->stall_enabled & BIT(idx))
37 reg |= ARM_SMMU_SCTLR_CFCFG;
38
39 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
40 }
41
qcom_adreno_smmu_get_fault_info(const void * cookie,struct adreno_smmu_fault_info * info)42 static void qcom_adreno_smmu_get_fault_info(const void *cookie,
43 struct adreno_smmu_fault_info *info)
44 {
45 struct arm_smmu_domain *smmu_domain = (void *)cookie;
46 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
47 struct arm_smmu_device *smmu = smmu_domain->smmu;
48
49 info->fsr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSR);
50 info->fsynr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR0);
51 info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1);
52 info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR);
53 info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
54 info->ttbr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
55 info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR);
56 }
57
qcom_adreno_smmu_set_stall(const void * cookie,bool enabled)58 static void qcom_adreno_smmu_set_stall(const void *cookie, bool enabled)
59 {
60 struct arm_smmu_domain *smmu_domain = (void *)cookie;
61 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
62 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu_domain->smmu);
63
64 if (enabled)
65 qsmmu->stall_enabled |= BIT(cfg->cbndx);
66 else
67 qsmmu->stall_enabled &= ~BIT(cfg->cbndx);
68 }
69
qcom_adreno_smmu_resume_translation(const void * cookie,bool terminate)70 static void qcom_adreno_smmu_resume_translation(const void *cookie, bool terminate)
71 {
72 struct arm_smmu_domain *smmu_domain = (void *)cookie;
73 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
74 struct arm_smmu_device *smmu = smmu_domain->smmu;
75 u32 reg = 0;
76
77 if (terminate)
78 reg |= ARM_SMMU_RESUME_TERMINATE;
79
80 arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_RESUME, reg);
81 }
82
83 #define QCOM_ADRENO_SMMU_GPU_SID 0
84
qcom_adreno_smmu_is_gpu_device(struct device * dev)85 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev)
86 {
87 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
88 int i;
89
90 /*
91 * The GPU will always use SID 0 so that is a handy way to uniquely
92 * identify it and configure it for per-instance pagetables
93 */
94 for (i = 0; i < fwspec->num_ids; i++) {
95 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
96
97 if (sid == QCOM_ADRENO_SMMU_GPU_SID)
98 return true;
99 }
100
101 return false;
102 }
103
qcom_adreno_smmu_get_ttbr1_cfg(const void * cookie)104 static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg(
105 const void *cookie)
106 {
107 struct arm_smmu_domain *smmu_domain = (void *)cookie;
108 struct io_pgtable *pgtable =
109 io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
110 return &pgtable->cfg;
111 }
112
113 /*
114 * Local implementation to configure TTBR0 with the specified pagetable config.
115 * The GPU driver will call this to enable TTBR0 when per-instance pagetables
116 * are active
117 */
118
qcom_adreno_smmu_set_ttbr0_cfg(const void * cookie,const struct io_pgtable_cfg * pgtbl_cfg)119 static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
120 const struct io_pgtable_cfg *pgtbl_cfg)
121 {
122 struct arm_smmu_domain *smmu_domain = (void *)cookie;
123 struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
124 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
125 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
126
127 /* The domain must have split pagetables already enabled */
128 if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
129 return -EINVAL;
130
131 /* If the pagetable config is NULL, disable TTBR0 */
132 if (!pgtbl_cfg) {
133 /* Do nothing if it is already disabled */
134 if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0))
135 return -EINVAL;
136
137 /* Set TCR to the original configuration */
138 cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg);
139 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
140 } else {
141 u32 tcr = cb->tcr[0];
142
143 /* Don't call this again if TTBR0 is already enabled */
144 if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0))
145 return -EINVAL;
146
147 tcr |= arm_smmu_lpae_tcr(pgtbl_cfg);
148 tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1);
149
150 cb->tcr[0] = tcr;
151 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
152 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
153 }
154
155 arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);
156
157 return 0;
158 }
159
qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,int start)160 static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
161 struct arm_smmu_device *smmu,
162 struct device *dev, int start)
163 {
164 int count;
165
166 /*
167 * Assign context bank 0 to the GPU device so the GPU hardware can
168 * switch pagetables
169 */
170 if (qcom_adreno_smmu_is_gpu_device(dev)) {
171 start = 0;
172 count = 1;
173 } else {
174 start = 1;
175 count = smmu->num_context_banks;
176 }
177
178 return __arm_smmu_alloc_bitmap(smmu->context_map, start, count);
179 }
180
qcom_adreno_can_do_ttbr1(struct arm_smmu_device * smmu)181 static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
182 {
183 const struct device_node *np = smmu->dev->of_node;
184
185 if (of_device_is_compatible(np, "qcom,msm8996-smmu-v2"))
186 return false;
187
188 return true;
189 }
190
qcom_adreno_smmu_init_context(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg,struct device * dev)191 static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
192 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
193 {
194 struct adreno_smmu_priv *priv;
195
196 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
197
198 /* Only enable split pagetables for the GPU device (SID 0) */
199 if (!qcom_adreno_smmu_is_gpu_device(dev))
200 return 0;
201
202 /*
203 * All targets that use the qcom,adreno-smmu compatible string *should*
204 * be AARCH64 stage 1 but double check because the arm-smmu code assumes
205 * that is the case when the TTBR1 quirk is enabled
206 */
207 if (qcom_adreno_can_do_ttbr1(smmu_domain->smmu) &&
208 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) &&
209 (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64))
210 pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1;
211
212 /*
213 * Initialize private interface with GPU:
214 */
215
216 priv = dev_get_drvdata(dev);
217 priv->cookie = smmu_domain;
218 priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg;
219 priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg;
220 priv->get_fault_info = qcom_adreno_smmu_get_fault_info;
221 priv->set_stall = qcom_adreno_smmu_set_stall;
222 priv->resume_translation = qcom_adreno_smmu_resume_translation;
223
224 return 0;
225 }
226
227 static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
228 { .compatible = "qcom,adreno" },
229 { .compatible = "qcom,mdp4" },
230 { .compatible = "qcom,mdss" },
231 { .compatible = "qcom,sc7180-mdss" },
232 { .compatible = "qcom,sc7180-mss-pil" },
233 { .compatible = "qcom,sc7280-mdss" },
234 { .compatible = "qcom,sc8180x-mdss" },
235 { .compatible = "qcom,sdm845-mdss" },
236 { .compatible = "qcom,sdm845-mss-pil" },
237 { }
238 };
239
qcom_smmu_init_context(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg,struct device * dev)240 static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
241 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
242 {
243 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
244
245 return 0;
246 }
247
qcom_smmu_cfg_probe(struct arm_smmu_device * smmu)248 static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu)
249 {
250 unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1);
251 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
252 u32 reg;
253 u32 smr;
254 int i;
255
256 /*
257 * With some firmware versions writes to S2CR of type FAULT are
258 * ignored, and writing BYPASS will end up written as FAULT in the
259 * register. Perform a write to S2CR to detect if this is the case and
260 * if so reserve a context bank to emulate bypass streams.
261 */
262 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) |
263 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) |
264 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT);
265 arm_smmu_gr0_write(smmu, last_s2cr, reg);
266 reg = arm_smmu_gr0_read(smmu, last_s2cr);
267 if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) {
268 qsmmu->bypass_quirk = true;
269 qsmmu->bypass_cbndx = smmu->num_context_banks - 1;
270
271 set_bit(qsmmu->bypass_cbndx, smmu->context_map);
272
273 arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0);
274
275 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS);
276 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg);
277 }
278
279 for (i = 0; i < smmu->num_mapping_groups; i++) {
280 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
281
282 if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) {
283 /* Ignore valid bit for SMR mask extraction. */
284 smr &= ~ARM_SMMU_SMR_VALID;
285 smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
286 smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
287 smmu->smrs[i].valid = true;
288
289 smmu->s2crs[i].type = S2CR_TYPE_BYPASS;
290 smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT;
291 smmu->s2crs[i].cbndx = 0xff;
292 }
293 }
294
295 return 0;
296 }
297
qcom_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)298 static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
299 {
300 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
301 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
302 u32 cbndx = s2cr->cbndx;
303 u32 type = s2cr->type;
304 u32 reg;
305
306 if (qsmmu->bypass_quirk) {
307 if (type == S2CR_TYPE_BYPASS) {
308 /*
309 * Firmware with quirky S2CR handling will substitute
310 * BYPASS writes with FAULT, so point the stream to the
311 * reserved context bank and ask for translation on the
312 * stream
313 */
314 type = S2CR_TYPE_TRANS;
315 cbndx = qsmmu->bypass_cbndx;
316 } else if (type == S2CR_TYPE_FAULT) {
317 /*
318 * Firmware with quirky S2CR handling will ignore FAULT
319 * writes, so trick it to write FAULT by asking for a
320 * BYPASS.
321 */
322 type = S2CR_TYPE_BYPASS;
323 cbndx = 0xff;
324 }
325 }
326
327 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) |
328 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) |
329 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
330 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
331 }
332
qcom_smmu_def_domain_type(struct device * dev)333 static int qcom_smmu_def_domain_type(struct device *dev)
334 {
335 const struct of_device_id *match =
336 of_match_device(qcom_smmu_client_of_match, dev);
337
338 return match ? IOMMU_DOMAIN_IDENTITY : 0;
339 }
340
qcom_sdm845_smmu500_reset(struct arm_smmu_device * smmu)341 static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
342 {
343 int ret;
344
345 /*
346 * To address performance degradation in non-real time clients,
347 * such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
348 * such as MTP and db845, whose firmwares implement secure monitor
349 * call handlers to turn on/off the wait-for-safe logic.
350 */
351 ret = qcom_scm_qsmmu500_wait_safe_toggle(0);
352 if (ret)
353 dev_warn(smmu->dev, "Failed to turn off SAFE logic\n");
354
355 return ret;
356 }
357
qcom_smmu500_reset(struct arm_smmu_device * smmu)358 static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
359 {
360 const struct device_node *np = smmu->dev->of_node;
361
362 arm_mmu500_reset(smmu);
363
364 if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
365 return qcom_sdm845_smmu500_reset(smmu);
366
367 return 0;
368 }
369
370 static const struct arm_smmu_impl qcom_smmu_impl = {
371 .init_context = qcom_smmu_init_context,
372 .cfg_probe = qcom_smmu_cfg_probe,
373 .def_domain_type = qcom_smmu_def_domain_type,
374 .reset = qcom_smmu500_reset,
375 .write_s2cr = qcom_smmu_write_s2cr,
376 };
377
378 static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
379 .init_context = qcom_adreno_smmu_init_context,
380 .def_domain_type = qcom_smmu_def_domain_type,
381 .reset = qcom_smmu500_reset,
382 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
383 .write_sctlr = qcom_adreno_smmu_write_sctlr,
384 };
385
qcom_smmu_create(struct arm_smmu_device * smmu,const struct arm_smmu_impl * impl)386 static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
387 const struct arm_smmu_impl *impl)
388 {
389 struct qcom_smmu *qsmmu;
390
391 /* Check to make sure qcom_scm has finished probing */
392 if (!qcom_scm_is_available())
393 return ERR_PTR(-EPROBE_DEFER);
394
395 qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL);
396 if (!qsmmu)
397 return ERR_PTR(-ENOMEM);
398
399 qsmmu->smmu.impl = impl;
400
401 return &qsmmu->smmu;
402 }
403
404 static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
405 { .compatible = "qcom,msm8998-smmu-v2" },
406 { .compatible = "qcom,sc7180-smmu-500" },
407 { .compatible = "qcom,sc7280-smmu-500" },
408 { .compatible = "qcom,sc8180x-smmu-500" },
409 { .compatible = "qcom,sdm630-smmu-v2" },
410 { .compatible = "qcom,sdm845-smmu-500" },
411 { .compatible = "qcom,sm6125-smmu-500" },
412 { .compatible = "qcom,sm8150-smmu-500" },
413 { .compatible = "qcom,sm8250-smmu-500" },
414 { .compatible = "qcom,sm8350-smmu-500" },
415 { }
416 };
417
418 #ifdef CONFIG_ACPI
419 static struct acpi_platform_list qcom_acpi_platlist[] = {
420 { "LENOVO", "CB-01 ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
421 { "QCOM ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
422 { }
423 };
424 #endif
425
qcom_smmu_impl_init(struct arm_smmu_device * smmu)426 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
427 {
428 const struct device_node *np = smmu->dev->of_node;
429
430 #ifdef CONFIG_ACPI
431 if (np == NULL) {
432 /* Match platform for ACPI boot */
433 if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
434 return qcom_smmu_create(smmu, &qcom_smmu_impl);
435 }
436 #endif
437
438 /*
439 * Do not change this order of implementation, i.e., first adreno
440 * smmu impl and then apss smmu since we can have both implementing
441 * arm,mmu-500 in which case we will miss setting adreno smmu specific
442 * features if the order is changed.
443 */
444 if (of_device_is_compatible(np, "qcom,adreno-smmu"))
445 return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
446
447 if (of_match_node(qcom_smmu_impl_of_match, np))
448 return qcom_smmu_create(smmu, &qcom_smmu_impl);
449
450 return smmu;
451 }
452
