1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017 Cadence
3 // Cadence PCIe host controller driver.
4 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
5
6 #include <linux/delay.h>
7 #include <linux/kernel.h>
8 #include <linux/list_sort.h>
9 #include <linux/of_address.h>
10 #include <linux/of_pci.h>
11 #include <linux/platform_device.h>
12
13 #include "pcie-cadence.h"
14
15 static u64 bar_max_size[] = {
16 [RP_BAR0] = _ULL(128 * SZ_2G),
17 [RP_BAR1] = SZ_2G,
18 [RP_NO_BAR] = _BITULL(63),
19 };
20
21 static u8 bar_aperture_mask[] = {
22 [RP_BAR0] = 0x1F,
23 [RP_BAR1] = 0xF,
24 };
25
cdns_pci_map_bus(struct pci_bus * bus,unsigned int devfn,int where)26 void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
27 int where)
28 {
29 struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
30 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
31 struct cdns_pcie *pcie = &rc->pcie;
32 unsigned int busn = bus->number;
33 u32 addr0, desc0;
34
35 if (pci_is_root_bus(bus)) {
36 /*
37 * Only the root port (devfn == 0) is connected to this bus.
38 * All other PCI devices are behind some bridge hence on another
39 * bus.
40 */
41 if (devfn)
42 return NULL;
43
44 return pcie->reg_base + (where & 0xfff);
45 }
46 /* Check that the link is up */
47 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))
48 return NULL;
49 /* Clear AXI link-down status */
50 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);
51
52 /* Update Output registers for AXI region 0. */
53 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |
54 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |
55 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);
56 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);
57
58 /* Configuration Type 0 or Type 1 access. */
59 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |
60 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);
61 /*
62 * The bus number was already set once for all in desc1 by
63 * cdns_pcie_host_init_address_translation().
64 */
65 if (busn == bridge->busnr + 1)
66 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;
67 else
68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;
69 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);
70
71 return rc->cfg_base + (where & 0xfff);
72 }
73
74 static struct pci_ops cdns_pcie_host_ops = {
75 .map_bus = cdns_pci_map_bus,
76 .read = pci_generic_config_read,
77 .write = pci_generic_config_write,
78 };
79
cdns_pcie_host_wait_for_link(struct cdns_pcie * pcie)80 static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)
81 {
82 struct device *dev = pcie->dev;
83 int retries;
84
85 /* Check if the link is up or not */
86 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
87 if (cdns_pcie_link_up(pcie)) {
88 dev_info(dev, "Link up\n");
89 return 0;
90 }
91 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
92 }
93
94 return -ETIMEDOUT;
95 }
96
cdns_pcie_retrain(struct cdns_pcie * pcie)97 static int cdns_pcie_retrain(struct cdns_pcie *pcie)
98 {
99 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
100 u16 lnk_stat, lnk_ctl;
101 int ret = 0;
102
103 /*
104 * Set retrain bit if current speed is 2.5 GB/s,
105 * but the PCIe root port support is > 2.5 GB/s.
106 */
107
108 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +
109 PCI_EXP_LNKCAP));
110 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
111 return ret;
112
113 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
114 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
115 lnk_ctl = cdns_pcie_rp_readw(pcie,
116 pcie_cap_off + PCI_EXP_LNKCTL);
117 lnk_ctl |= PCI_EXP_LNKCTL_RL;
118 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,
119 lnk_ctl);
120
121 ret = cdns_pcie_host_wait_for_link(pcie);
122 }
123 return ret;
124 }
125
cdns_pcie_host_enable_ptm_response(struct cdns_pcie * pcie)126 static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie)
127 {
128 u32 val;
129
130 val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);
131 cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);
132 }
133
cdns_pcie_host_start_link(struct cdns_pcie_rc * rc)134 static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc)
135 {
136 struct cdns_pcie *pcie = &rc->pcie;
137 int ret;
138
139 ret = cdns_pcie_host_wait_for_link(pcie);
140
141 /*
142 * Retrain link for Gen2 training defect
143 * if quirk flag is set.
144 */
145 if (!ret && rc->quirk_retrain_flag)
146 ret = cdns_pcie_retrain(pcie);
147
148 return ret;
149 }
150
cdns_pcie_host_init_root_port(struct cdns_pcie_rc * rc)151 static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)
152 {
153 struct cdns_pcie *pcie = &rc->pcie;
154 u32 value, ctrl;
155 u32 id;
156
157 /*
158 * Set the root complex BAR configuration register:
159 * - disable both BAR0 and BAR1.
160 * - enable Prefetchable Memory Base and Limit registers in type 1
161 * config space (64 bits).
162 * - enable IO Base and Limit registers in type 1 config
163 * space (32 bits).
164 */
165 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
166 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |
167 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |
168 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |
169 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |
170 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |
171 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;
172 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
173
174 /* Set root port configuration space */
175 if (rc->vendor_id != 0xffff) {
176 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
177 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
178 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
179 }
180
181 if (rc->device_id != 0xffff)
182 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
183
184 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);
185 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);
186 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
187
188 return 0;
189 }
190
cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc * rc,enum cdns_pcie_rp_bar bar,u64 cpu_addr,u64 size,unsigned long flags)191 static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,
192 enum cdns_pcie_rp_bar bar,
193 u64 cpu_addr, u64 size,
194 unsigned long flags)
195 {
196 struct cdns_pcie *pcie = &rc->pcie;
197 u32 addr0, addr1, aperture, value;
198
199 if (!rc->avail_ib_bar[bar])
200 return -EBUSY;
201
202 rc->avail_ib_bar[bar] = false;
203
204 aperture = ilog2(size);
205 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |
206 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
207 addr1 = upper_32_bits(cpu_addr);
208 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);
209 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);
210
211 if (bar == RP_NO_BAR)
212 return 0;
213
214 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);
215 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |
216 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |
217 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |
218 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |
219 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));
220 if (size + cpu_addr >= SZ_4G) {
221 if (!(flags & IORESOURCE_PREFETCH))
222 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);
223 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);
224 } else {
225 if (!(flags & IORESOURCE_PREFETCH))
226 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);
227 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);
228 }
229
230 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);
231 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
232
233 return 0;
234 }
235
236 static enum cdns_pcie_rp_bar
cdns_pcie_host_find_min_bar(struct cdns_pcie_rc * rc,u64 size)237 cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)
238 {
239 enum cdns_pcie_rp_bar bar, sel_bar;
240
241 sel_bar = RP_BAR_UNDEFINED;
242 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
243 if (!rc->avail_ib_bar[bar])
244 continue;
245
246 if (size <= bar_max_size[bar]) {
247 if (sel_bar == RP_BAR_UNDEFINED) {
248 sel_bar = bar;
249 continue;
250 }
251
252 if (bar_max_size[bar] < bar_max_size[sel_bar])
253 sel_bar = bar;
254 }
255 }
256
257 return sel_bar;
258 }
259
260 static enum cdns_pcie_rp_bar
cdns_pcie_host_find_max_bar(struct cdns_pcie_rc * rc,u64 size)261 cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)
262 {
263 enum cdns_pcie_rp_bar bar, sel_bar;
264
265 sel_bar = RP_BAR_UNDEFINED;
266 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
267 if (!rc->avail_ib_bar[bar])
268 continue;
269
270 if (size >= bar_max_size[bar]) {
271 if (sel_bar == RP_BAR_UNDEFINED) {
272 sel_bar = bar;
273 continue;
274 }
275
276 if (bar_max_size[bar] > bar_max_size[sel_bar])
277 sel_bar = bar;
278 }
279 }
280
281 return sel_bar;
282 }
283
cdns_pcie_host_bar_config(struct cdns_pcie_rc * rc,struct resource_entry * entry)284 static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
285 struct resource_entry *entry)
286 {
287 u64 cpu_addr, pci_addr, size, winsize;
288 struct cdns_pcie *pcie = &rc->pcie;
289 struct device *dev = pcie->dev;
290 enum cdns_pcie_rp_bar bar;
291 unsigned long flags;
292 int ret;
293
294 cpu_addr = entry->res->start;
295 pci_addr = entry->res->start - entry->offset;
296 flags = entry->res->flags;
297 size = resource_size(entry->res);
298
299 if (entry->offset) {
300 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",
301 pci_addr, cpu_addr);
302 return -EINVAL;
303 }
304
305 while (size > 0) {
306 /*
307 * Try to find a minimum BAR whose size is greater than
308 * or equal to the remaining resource_entry size. This will
309 * fail if the size of each of the available BARs is less than
310 * the remaining resource_entry size.
311 * If a minimum BAR is found, IB ATU will be configured and
312 * exited.
313 */
314 bar = cdns_pcie_host_find_min_bar(rc, size);
315 if (bar != RP_BAR_UNDEFINED) {
316 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,
317 size, flags);
318 if (ret)
319 dev_err(dev, "IB BAR: %d config failed\n", bar);
320 return ret;
321 }
322
323 /*
324 * If the control reaches here, it would mean the remaining
325 * resource_entry size cannot be fitted in a single BAR. So we
326 * find a maximum BAR whose size is less than or equal to the
327 * remaining resource_entry size and split the resource entry
328 * so that part of resource entry is fitted inside the maximum
329 * BAR. The remaining size would be fitted during the next
330 * iteration of the loop.
331 * If a maximum BAR is not found, there is no way we can fit
332 * this resource_entry, so we error out.
333 */
334 bar = cdns_pcie_host_find_max_bar(rc, size);
335 if (bar == RP_BAR_UNDEFINED) {
336 dev_err(dev, "No free BAR to map cpu_addr %llx\n",
337 cpu_addr);
338 return -EINVAL;
339 }
340
341 winsize = bar_max_size[bar];
342 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,
343 flags);
344 if (ret) {
345 dev_err(dev, "IB BAR: %d config failed\n", bar);
346 return ret;
347 }
348
349 size -= winsize;
350 cpu_addr += winsize;
351 }
352
353 return 0;
354 }
355
cdns_pcie_host_dma_ranges_cmp(void * priv,const struct list_head * a,const struct list_head * b)356 static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,
357 const struct list_head *b)
358 {
359 struct resource_entry *entry1, *entry2;
360
361 entry1 = container_of(a, struct resource_entry, node);
362 entry2 = container_of(b, struct resource_entry, node);
363
364 return resource_size(entry2->res) - resource_size(entry1->res);
365 }
366
cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc * rc)367 static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)
368 {
369 struct cdns_pcie *pcie = &rc->pcie;
370 struct device *dev = pcie->dev;
371 struct device_node *np = dev->of_node;
372 struct pci_host_bridge *bridge;
373 struct resource_entry *entry;
374 u32 no_bar_nbits = 32;
375 int err;
376
377 bridge = pci_host_bridge_from_priv(rc);
378 if (!bridge)
379 return -ENOMEM;
380
381 if (list_empty(&bridge->dma_ranges)) {
382 of_property_read_u32(np, "cdns,no-bar-match-nbits",
383 &no_bar_nbits);
384 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,
385 (u64)1 << no_bar_nbits, 0);
386 if (err)
387 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);
388 return err;
389 }
390
391 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);
392
393 resource_list_for_each_entry(entry, &bridge->dma_ranges) {
394 err = cdns_pcie_host_bar_config(rc, entry);
395 if (err) {
396 dev_err(dev, "Fail to configure IB using dma-ranges\n");
397 return err;
398 }
399 }
400
401 return 0;
402 }
403
cdns_pcie_host_init_address_translation(struct cdns_pcie_rc * rc)404 static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
405 {
406 struct cdns_pcie *pcie = &rc->pcie;
407 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);
408 struct resource *cfg_res = rc->cfg_res;
409 struct resource_entry *entry;
410 u64 cpu_addr = cfg_res->start;
411 u32 addr0, addr1, desc1;
412 int r, busnr = 0;
413
414 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);
415 if (entry)
416 busnr = entry->res->start;
417
418 /*
419 * Reserve region 0 for PCI configure space accesses:
420 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by
421 * cdns_pci_map_bus(), other region registers are set here once for all.
422 */
423 addr1 = 0; /* Should be programmed to zero. */
424 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);
425 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
426 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
427
428 if (pcie->ops->cpu_addr_fixup)
429 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);
430
431 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
432 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
433 addr1 = upper_32_bits(cpu_addr);
434 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);
435 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);
436
437 r = 1;
438 resource_list_for_each_entry(entry, &bridge->windows) {
439 struct resource *res = entry->res;
440 u64 pci_addr = res->start - entry->offset;
441
442 if (resource_type(res) == IORESOURCE_IO)
443 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
444 true,
445 pci_pio_to_address(res->start),
446 pci_addr,
447 resource_size(res));
448 else
449 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
450 false,
451 res->start,
452 pci_addr,
453 resource_size(res));
454
455 r++;
456 }
457
458 return cdns_pcie_host_map_dma_ranges(rc);
459 }
460
cdns_pcie_host_init(struct device * dev,struct cdns_pcie_rc * rc)461 static int cdns_pcie_host_init(struct device *dev,
462 struct cdns_pcie_rc *rc)
463 {
464 int err;
465
466 err = cdns_pcie_host_init_root_port(rc);
467 if (err)
468 return err;
469
470 return cdns_pcie_host_init_address_translation(rc);
471 }
472
cdns_pcie_host_setup(struct cdns_pcie_rc * rc)473 int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
474 {
475 struct device *dev = rc->pcie.dev;
476 struct platform_device *pdev = to_platform_device(dev);
477 struct device_node *np = dev->of_node;
478 struct pci_host_bridge *bridge;
479 enum cdns_pcie_rp_bar bar;
480 struct cdns_pcie *pcie;
481 struct resource *res;
482 int ret;
483
484 bridge = pci_host_bridge_from_priv(rc);
485 if (!bridge)
486 return -ENOMEM;
487
488 pcie = &rc->pcie;
489 pcie->is_rc = true;
490
491 rc->vendor_id = 0xffff;
492 of_property_read_u32(np, "vendor-id", &rc->vendor_id);
493
494 rc->device_id = 0xffff;
495 of_property_read_u32(np, "device-id", &rc->device_id);
496
497 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");
498 if (IS_ERR(pcie->reg_base)) {
499 dev_err(dev, "missing \"reg\"\n");
500 return PTR_ERR(pcie->reg_base);
501 }
502
503 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
504 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);
505 if (IS_ERR(rc->cfg_base))
506 return PTR_ERR(rc->cfg_base);
507 rc->cfg_res = res;
508
509 if (rc->quirk_detect_quiet_flag)
510 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);
511
512 cdns_pcie_host_enable_ptm_response(pcie);
513
514 ret = cdns_pcie_start_link(pcie);
515 if (ret) {
516 dev_err(dev, "Failed to start link\n");
517 return ret;
518 }
519
520 ret = cdns_pcie_host_start_link(rc);
521 if (ret)
522 dev_dbg(dev, "PCIe link never came up\n");
523
524 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)
525 rc->avail_ib_bar[bar] = true;
526
527 ret = cdns_pcie_host_init(dev, rc);
528 if (ret)
529 return ret;
530
531 if (!bridge->ops)
532 bridge->ops = &cdns_pcie_host_ops;
533
534 ret = pci_host_probe(bridge);
535 if (ret < 0)
536 goto err_init;
537
538 return 0;
539
540 err_init:
541 pm_runtime_put_sync(dev);
542
543 return ret;
544 }
545
