1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/io-64-nonatomic-hi-lo.h>
4 #include <linux/seq_file.h>
5 #include <linux/device.h>
6 #include <linux/delay.h>
7
8 #include "cxlmem.h"
9 #include "core.h"
10
11 /**
12 * DOC: cxl core hdm
13 *
14 * Compute Express Link Host Managed Device Memory, starting with the
15 * CXL 2.0 specification, is managed by an array of HDM Decoder register
16 * instances per CXL port and per CXL endpoint. Define common helpers
17 * for enumerating these registers and capabilities.
18 */
19
20 DECLARE_RWSEM(cxl_dpa_rwsem);
21
add_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map)22 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
23 int *target_map)
24 {
25 int rc;
26
27 rc = cxl_decoder_add_locked(cxld, target_map);
28 if (rc) {
29 put_device(&cxld->dev);
30 dev_err(&port->dev, "Failed to add decoder\n");
31 return rc;
32 }
33
34 rc = cxl_decoder_autoremove(&port->dev, cxld);
35 if (rc)
36 return rc;
37
38 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
39
40 return 0;
41 }
42
43 /*
44 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
45 * single ported host-bridges need not publish a decoder capability when a
46 * passthrough decode can be assumed, i.e. all transactions that the uport sees
47 * are claimed and passed to the single dport. Disable the range until the first
48 * CXL region is enumerated / activated.
49 */
devm_cxl_add_passthrough_decoder(struct cxl_port * port)50 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
51 {
52 struct cxl_switch_decoder *cxlsd;
53 struct cxl_dport *dport = NULL;
54 int single_port_map[1];
55 unsigned long index;
56
57 cxlsd = cxl_switch_decoder_alloc(port, 1);
58 if (IS_ERR(cxlsd))
59 return PTR_ERR(cxlsd);
60
61 device_lock_assert(&port->dev);
62
63 xa_for_each(&port->dports, index, dport)
64 break;
65 single_port_map[0] = dport->port_id;
66
67 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
68 }
69 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
70
parse_hdm_decoder_caps(struct cxl_hdm * cxlhdm)71 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
72 {
73 u32 hdm_cap;
74
75 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
76 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
77 cxlhdm->target_count =
78 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
79 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
80 cxlhdm->interleave_mask |= GENMASK(11, 8);
81 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
82 cxlhdm->interleave_mask |= GENMASK(14, 12);
83 }
84
map_hdm_decoder_regs(struct cxl_port * port,void __iomem * crb)85 static void __iomem *map_hdm_decoder_regs(struct cxl_port *port,
86 void __iomem *crb)
87 {
88 struct cxl_component_reg_map map;
89
90 cxl_probe_component_regs(&port->dev, crb, &map);
91 if (!map.hdm_decoder.valid) {
92 dev_err(&port->dev, "HDM decoder registers invalid\n");
93 return IOMEM_ERR_PTR(-ENXIO);
94 }
95
96 return crb + map.hdm_decoder.offset;
97 }
98
99 /**
100 * devm_cxl_setup_hdm - map HDM decoder component registers
101 * @port: cxl_port to map
102 */
devm_cxl_setup_hdm(struct cxl_port * port)103 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port)
104 {
105 struct device *dev = &port->dev;
106 void __iomem *crb, *hdm;
107 struct cxl_hdm *cxlhdm;
108
109 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
110 if (!cxlhdm)
111 return ERR_PTR(-ENOMEM);
112
113 cxlhdm->port = port;
114 crb = devm_cxl_iomap_block(dev, port->component_reg_phys,
115 CXL_COMPONENT_REG_BLOCK_SIZE);
116 if (!crb) {
117 dev_err(dev, "No component registers mapped\n");
118 return ERR_PTR(-ENXIO);
119 }
120
121 hdm = map_hdm_decoder_regs(port, crb);
122 if (IS_ERR(hdm))
123 return ERR_CAST(hdm);
124 cxlhdm->regs.hdm_decoder = hdm;
125
126 parse_hdm_decoder_caps(cxlhdm);
127 if (cxlhdm->decoder_count == 0) {
128 dev_err(dev, "Spec violation. Caps invalid\n");
129 return ERR_PTR(-ENXIO);
130 }
131
132 dev_set_drvdata(dev, cxlhdm);
133
134 return cxlhdm;
135 }
136 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
137
__cxl_dpa_debug(struct seq_file * file,struct resource * r,int depth)138 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
139 {
140 unsigned long long start = r->start, end = r->end;
141
142 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
143 r->name);
144 }
145
cxl_dpa_debug(struct seq_file * file,struct cxl_dev_state * cxlds)146 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
147 {
148 struct resource *p1, *p2;
149
150 down_read(&cxl_dpa_rwsem);
151 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
152 __cxl_dpa_debug(file, p1, 0);
153 for (p2 = p1->child; p2; p2 = p2->sibling)
154 __cxl_dpa_debug(file, p2, 1);
155 }
156 up_read(&cxl_dpa_rwsem);
157 }
158 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
159
160 /*
161 * Must be called in a context that synchronizes against this decoder's
162 * port ->remove() callback (like an endpoint decoder sysfs attribute)
163 */
__cxl_dpa_release(struct cxl_endpoint_decoder * cxled)164 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
165 {
166 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
167 struct cxl_port *port = cxled_to_port(cxled);
168 struct cxl_dev_state *cxlds = cxlmd->cxlds;
169 struct resource *res = cxled->dpa_res;
170 resource_size_t skip_start;
171
172 lockdep_assert_held_write(&cxl_dpa_rwsem);
173
174 /* save @skip_start, before @res is released */
175 skip_start = res->start - cxled->skip;
176 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
177 if (cxled->skip)
178 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
179 cxled->skip = 0;
180 cxled->dpa_res = NULL;
181 put_device(&cxled->cxld.dev);
182 port->hdm_end--;
183 }
184
cxl_dpa_release(void * cxled)185 static void cxl_dpa_release(void *cxled)
186 {
187 down_write(&cxl_dpa_rwsem);
188 __cxl_dpa_release(cxled);
189 up_write(&cxl_dpa_rwsem);
190 }
191
192 /*
193 * Must be called from context that will not race port device
194 * unregistration, like decoder sysfs attribute methods
195 */
devm_cxl_dpa_release(struct cxl_endpoint_decoder * cxled)196 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
197 {
198 struct cxl_port *port = cxled_to_port(cxled);
199
200 lockdep_assert_held_write(&cxl_dpa_rwsem);
201 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
202 __cxl_dpa_release(cxled);
203 }
204
__cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)205 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
206 resource_size_t base, resource_size_t len,
207 resource_size_t skipped)
208 {
209 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
210 struct cxl_port *port = cxled_to_port(cxled);
211 struct cxl_dev_state *cxlds = cxlmd->cxlds;
212 struct device *dev = &port->dev;
213 struct resource *res;
214
215 lockdep_assert_held_write(&cxl_dpa_rwsem);
216
217 if (!len)
218 goto success;
219
220 if (cxled->dpa_res) {
221 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
222 port->id, cxled->cxld.id, cxled->dpa_res);
223 return -EBUSY;
224 }
225
226 if (port->hdm_end + 1 != cxled->cxld.id) {
227 /*
228 * Assumes alloc and commit order is always in hardware instance
229 * order per expectations from 8.2.5.12.20 Committing Decoder
230 * Programming that enforce decoder[m] committed before
231 * decoder[m+1] commit start.
232 */
233 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
234 cxled->cxld.id, port->id, port->hdm_end + 1);
235 return -EBUSY;
236 }
237
238 if (skipped) {
239 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
240 dev_name(&cxled->cxld.dev), 0);
241 if (!res) {
242 dev_dbg(dev,
243 "decoder%d.%d: failed to reserve skipped space\n",
244 port->id, cxled->cxld.id);
245 return -EBUSY;
246 }
247 }
248 res = __request_region(&cxlds->dpa_res, base, len,
249 dev_name(&cxled->cxld.dev), 0);
250 if (!res) {
251 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
252 port->id, cxled->cxld.id);
253 if (skipped)
254 __release_region(&cxlds->dpa_res, base - skipped,
255 skipped);
256 return -EBUSY;
257 }
258 cxled->dpa_res = res;
259 cxled->skip = skipped;
260
261 if (resource_contains(&cxlds->pmem_res, res))
262 cxled->mode = CXL_DECODER_PMEM;
263 else if (resource_contains(&cxlds->ram_res, res))
264 cxled->mode = CXL_DECODER_RAM;
265 else {
266 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
267 cxled->cxld.id, cxled->dpa_res);
268 cxled->mode = CXL_DECODER_MIXED;
269 }
270
271 success:
272 port->hdm_end++;
273 get_device(&cxled->cxld.dev);
274 return 0;
275 }
276
devm_cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)277 static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
278 resource_size_t base, resource_size_t len,
279 resource_size_t skipped)
280 {
281 struct cxl_port *port = cxled_to_port(cxled);
282 int rc;
283
284 down_write(&cxl_dpa_rwsem);
285 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
286 up_write(&cxl_dpa_rwsem);
287
288 if (rc)
289 return rc;
290
291 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
292 }
293
cxl_dpa_size(struct cxl_endpoint_decoder * cxled)294 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
295 {
296 resource_size_t size = 0;
297
298 down_read(&cxl_dpa_rwsem);
299 if (cxled->dpa_res)
300 size = resource_size(cxled->dpa_res);
301 up_read(&cxl_dpa_rwsem);
302
303 return size;
304 }
305
cxl_dpa_resource_start(struct cxl_endpoint_decoder * cxled)306 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
307 {
308 resource_size_t base = -1;
309
310 down_read(&cxl_dpa_rwsem);
311 if (cxled->dpa_res)
312 base = cxled->dpa_res->start;
313 up_read(&cxl_dpa_rwsem);
314
315 return base;
316 }
317
cxl_dpa_free(struct cxl_endpoint_decoder * cxled)318 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
319 {
320 struct cxl_port *port = cxled_to_port(cxled);
321 struct device *dev = &cxled->cxld.dev;
322 int rc;
323
324 down_write(&cxl_dpa_rwsem);
325 if (!cxled->dpa_res) {
326 rc = 0;
327 goto out;
328 }
329 if (cxled->cxld.region) {
330 dev_dbg(dev, "decoder assigned to: %s\n",
331 dev_name(&cxled->cxld.region->dev));
332 rc = -EBUSY;
333 goto out;
334 }
335 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
336 dev_dbg(dev, "decoder enabled\n");
337 rc = -EBUSY;
338 goto out;
339 }
340 if (cxled->cxld.id != port->hdm_end) {
341 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
342 port->hdm_end);
343 rc = -EBUSY;
344 goto out;
345 }
346 devm_cxl_dpa_release(cxled);
347 rc = 0;
348 out:
349 up_write(&cxl_dpa_rwsem);
350 return rc;
351 }
352
cxl_dpa_set_mode(struct cxl_endpoint_decoder * cxled,enum cxl_decoder_mode mode)353 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
354 enum cxl_decoder_mode mode)
355 {
356 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
357 struct cxl_dev_state *cxlds = cxlmd->cxlds;
358 struct device *dev = &cxled->cxld.dev;
359 int rc;
360
361 switch (mode) {
362 case CXL_DECODER_RAM:
363 case CXL_DECODER_PMEM:
364 break;
365 default:
366 dev_dbg(dev, "unsupported mode: %d\n", mode);
367 return -EINVAL;
368 }
369
370 down_write(&cxl_dpa_rwsem);
371 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
372 rc = -EBUSY;
373 goto out;
374 }
375
376 /*
377 * Only allow modes that are supported by the current partition
378 * configuration
379 */
380 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
381 dev_dbg(dev, "no available pmem capacity\n");
382 rc = -ENXIO;
383 goto out;
384 }
385 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
386 dev_dbg(dev, "no available ram capacity\n");
387 rc = -ENXIO;
388 goto out;
389 }
390
391 cxled->mode = mode;
392 rc = 0;
393 out:
394 up_write(&cxl_dpa_rwsem);
395
396 return rc;
397 }
398
cxl_dpa_alloc(struct cxl_endpoint_decoder * cxled,unsigned long long size)399 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
400 {
401 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
402 resource_size_t free_ram_start, free_pmem_start;
403 struct cxl_port *port = cxled_to_port(cxled);
404 struct cxl_dev_state *cxlds = cxlmd->cxlds;
405 struct device *dev = &cxled->cxld.dev;
406 resource_size_t start, avail, skip;
407 struct resource *p, *last;
408 int rc;
409
410 down_write(&cxl_dpa_rwsem);
411 if (cxled->cxld.region) {
412 dev_dbg(dev, "decoder attached to %s\n",
413 dev_name(&cxled->cxld.region->dev));
414 rc = -EBUSY;
415 goto out;
416 }
417
418 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
419 dev_dbg(dev, "decoder enabled\n");
420 rc = -EBUSY;
421 goto out;
422 }
423
424 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
425 last = p;
426 if (last)
427 free_ram_start = last->end + 1;
428 else
429 free_ram_start = cxlds->ram_res.start;
430
431 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
432 last = p;
433 if (last)
434 free_pmem_start = last->end + 1;
435 else
436 free_pmem_start = cxlds->pmem_res.start;
437
438 if (cxled->mode == CXL_DECODER_RAM) {
439 start = free_ram_start;
440 avail = cxlds->ram_res.end - start + 1;
441 skip = 0;
442 } else if (cxled->mode == CXL_DECODER_PMEM) {
443 resource_size_t skip_start, skip_end;
444
445 start = free_pmem_start;
446 avail = cxlds->pmem_res.end - start + 1;
447 skip_start = free_ram_start;
448
449 /*
450 * If some pmem is already allocated, then that allocation
451 * already handled the skip.
452 */
453 if (cxlds->pmem_res.child &&
454 skip_start == cxlds->pmem_res.child->start)
455 skip_end = skip_start - 1;
456 else
457 skip_end = start - 1;
458 skip = skip_end - skip_start + 1;
459 } else {
460 dev_dbg(dev, "mode not set\n");
461 rc = -EINVAL;
462 goto out;
463 }
464
465 if (size > avail) {
466 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
467 cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
468 &avail);
469 rc = -ENOSPC;
470 goto out;
471 }
472
473 rc = __cxl_dpa_reserve(cxled, start, size, skip);
474 out:
475 up_write(&cxl_dpa_rwsem);
476
477 if (rc)
478 return rc;
479
480 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
481 }
482
cxld_set_interleave(struct cxl_decoder * cxld,u32 * ctrl)483 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
484 {
485 u16 eig;
486 u8 eiw;
487
488 /*
489 * Input validation ensures these warns never fire, but otherwise
490 * suppress unititalized variable usage warnings.
491 */
492 if (WARN_ONCE(ways_to_cxl(cxld->interleave_ways, &eiw),
493 "invalid interleave_ways: %d\n", cxld->interleave_ways))
494 return;
495 if (WARN_ONCE(granularity_to_cxl(cxld->interleave_granularity, &eig),
496 "invalid interleave_granularity: %d\n",
497 cxld->interleave_granularity))
498 return;
499
500 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
501 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
502 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
503 }
504
cxld_set_type(struct cxl_decoder * cxld,u32 * ctrl)505 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
506 {
507 u32p_replace_bits(ctrl, !!(cxld->target_type == 3),
508 CXL_HDM_DECODER0_CTRL_TYPE);
509 }
510
cxlsd_set_targets(struct cxl_switch_decoder * cxlsd,u64 * tgt)511 static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
512 {
513 struct cxl_dport **t = &cxlsd->target[0];
514 int ways = cxlsd->cxld.interleave_ways;
515
516 if (dev_WARN_ONCE(&cxlsd->cxld.dev,
517 ways > 8 || ways > cxlsd->nr_targets,
518 "ways: %d overflows targets: %d\n", ways,
519 cxlsd->nr_targets))
520 return -ENXIO;
521
522 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
523 if (ways > 1)
524 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
525 if (ways > 2)
526 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
527 if (ways > 3)
528 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
529 if (ways > 4)
530 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
531 if (ways > 5)
532 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
533 if (ways > 6)
534 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
535 if (ways > 7)
536 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
537
538 return 0;
539 }
540
541 /*
542 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
543 * committed or error within 10ms, but just be generous with 20ms to account for
544 * clock skew and other marginal behavior
545 */
546 #define COMMIT_TIMEOUT_MS 20
cxld_await_commit(void __iomem * hdm,int id)547 static int cxld_await_commit(void __iomem *hdm, int id)
548 {
549 u32 ctrl;
550 int i;
551
552 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
553 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
554 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
555 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
556 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
557 return -EIO;
558 }
559 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
560 return 0;
561 fsleep(1000);
562 }
563
564 return -ETIMEDOUT;
565 }
566
cxl_decoder_commit(struct cxl_decoder * cxld)567 static int cxl_decoder_commit(struct cxl_decoder *cxld)
568 {
569 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
570 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
571 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
572 int id = cxld->id, rc;
573 u64 base, size;
574 u32 ctrl;
575
576 if (cxld->flags & CXL_DECODER_F_ENABLE)
577 return 0;
578
579 if (port->commit_end + 1 != id) {
580 dev_dbg(&port->dev,
581 "%s: out of order commit, expected decoder%d.%d\n",
582 dev_name(&cxld->dev), port->id, port->commit_end + 1);
583 return -EBUSY;
584 }
585
586 down_read(&cxl_dpa_rwsem);
587 /* common decoder settings */
588 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
589 cxld_set_interleave(cxld, &ctrl);
590 cxld_set_type(cxld, &ctrl);
591 base = cxld->hpa_range.start;
592 size = range_len(&cxld->hpa_range);
593
594 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
595 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
596 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
597 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
598
599 if (is_switch_decoder(&cxld->dev)) {
600 struct cxl_switch_decoder *cxlsd =
601 to_cxl_switch_decoder(&cxld->dev);
602 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
603 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
604 u64 targets;
605
606 rc = cxlsd_set_targets(cxlsd, &targets);
607 if (rc) {
608 dev_dbg(&port->dev, "%s: target configuration error\n",
609 dev_name(&cxld->dev));
610 goto err;
611 }
612
613 writel(upper_32_bits(targets), tl_hi);
614 writel(lower_32_bits(targets), tl_lo);
615 } else {
616 struct cxl_endpoint_decoder *cxled =
617 to_cxl_endpoint_decoder(&cxld->dev);
618 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
619 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
620
621 writel(upper_32_bits(cxled->skip), sk_hi);
622 writel(lower_32_bits(cxled->skip), sk_lo);
623 }
624
625 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
626 up_read(&cxl_dpa_rwsem);
627
628 port->commit_end++;
629 rc = cxld_await_commit(hdm, cxld->id);
630 err:
631 if (rc) {
632 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
633 dev_name(&cxld->dev), rc);
634 cxld->reset(cxld);
635 return rc;
636 }
637 cxld->flags |= CXL_DECODER_F_ENABLE;
638
639 return 0;
640 }
641
cxl_decoder_reset(struct cxl_decoder * cxld)642 static int cxl_decoder_reset(struct cxl_decoder *cxld)
643 {
644 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
645 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
646 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
647 int id = cxld->id;
648 u32 ctrl;
649
650 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
651 return 0;
652
653 if (port->commit_end != id) {
654 dev_dbg(&port->dev,
655 "%s: out of order reset, expected decoder%d.%d\n",
656 dev_name(&cxld->dev), port->id, port->commit_end);
657 return -EBUSY;
658 }
659
660 down_read(&cxl_dpa_rwsem);
661 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
662 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
663 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
664
665 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
666 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
667 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
668 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
669 up_read(&cxl_dpa_rwsem);
670
671 port->commit_end--;
672 cxld->flags &= ~CXL_DECODER_F_ENABLE;
673
674 return 0;
675 }
676
init_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map,void __iomem * hdm,int which,u64 * dpa_base)677 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
678 int *target_map, void __iomem *hdm, int which,
679 u64 *dpa_base)
680 {
681 struct cxl_endpoint_decoder *cxled = NULL;
682 u64 size, base, skip, dpa_size;
683 bool committed;
684 u32 remainder;
685 int i, rc;
686 u32 ctrl;
687 union {
688 u64 value;
689 unsigned char target_id[8];
690 } target_list;
691
692 if (is_endpoint_decoder(&cxld->dev))
693 cxled = to_cxl_endpoint_decoder(&cxld->dev);
694
695 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
696 base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
697 size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
698 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
699 cxld->commit = cxl_decoder_commit;
700 cxld->reset = cxl_decoder_reset;
701
702 if (!committed)
703 size = 0;
704 if (base == U64_MAX || size == U64_MAX) {
705 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
706 port->id, cxld->id);
707 return -ENXIO;
708 }
709
710 cxld->hpa_range = (struct range) {
711 .start = base,
712 .end = base + size - 1,
713 };
714
715 /* decoders are enabled if committed */
716 if (committed) {
717 cxld->flags |= CXL_DECODER_F_ENABLE;
718 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
719 cxld->flags |= CXL_DECODER_F_LOCK;
720 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
721 cxld->target_type = CXL_DECODER_EXPANDER;
722 else
723 cxld->target_type = CXL_DECODER_ACCELERATOR;
724 if (cxld->id != port->commit_end + 1) {
725 dev_warn(&port->dev,
726 "decoder%d.%d: Committed out of order\n",
727 port->id, cxld->id);
728 return -ENXIO;
729 }
730 port->commit_end = cxld->id;
731 } else {
732 /* unless / until type-2 drivers arrive, assume type-3 */
733 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) {
734 ctrl |= CXL_HDM_DECODER0_CTRL_TYPE;
735 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
736 }
737 cxld->target_type = CXL_DECODER_EXPANDER;
738 }
739 rc = cxl_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
740 &cxld->interleave_ways);
741 if (rc) {
742 dev_warn(&port->dev,
743 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
744 port->id, cxld->id, ctrl);
745 return rc;
746 }
747 rc = cxl_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
748 &cxld->interleave_granularity);
749 if (rc)
750 return rc;
751
752 if (!cxled) {
753 target_list.value =
754 ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
755 for (i = 0; i < cxld->interleave_ways; i++)
756 target_map[i] = target_list.target_id[i];
757
758 return 0;
759 }
760
761 if (!committed)
762 return 0;
763
764 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
765 if (remainder) {
766 dev_err(&port->dev,
767 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
768 port->id, cxld->id, size, cxld->interleave_ways);
769 return -ENXIO;
770 }
771 skip = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
772 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
773 if (rc) {
774 dev_err(&port->dev,
775 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
776 port->id, cxld->id, *dpa_base,
777 *dpa_base + dpa_size + skip - 1, rc);
778 return rc;
779 }
780 *dpa_base += dpa_size + skip;
781 return 0;
782 }
783
784 /**
785 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
786 * @cxlhdm: Structure to populate with HDM capabilities
787 */
devm_cxl_enumerate_decoders(struct cxl_hdm * cxlhdm)788 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
789 {
790 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
791 struct cxl_port *port = cxlhdm->port;
792 int i, committed;
793 u64 dpa_base = 0;
794 u32 ctrl;
795
796 /*
797 * Since the register resource was recently claimed via request_region()
798 * be careful about trusting the "not-committed" status until the commit
799 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
800 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
801 * host and target.
802 */
803 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
804 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
805 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
806 committed++;
807 }
808
809 /* ensure that future checks of committed can be trusted */
810 if (committed != cxlhdm->decoder_count)
811 msleep(20);
812
813 for (i = 0; i < cxlhdm->decoder_count; i++) {
814 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
815 int rc, target_count = cxlhdm->target_count;
816 struct cxl_decoder *cxld;
817
818 if (is_cxl_endpoint(port)) {
819 struct cxl_endpoint_decoder *cxled;
820
821 cxled = cxl_endpoint_decoder_alloc(port);
822 if (IS_ERR(cxled)) {
823 dev_warn(&port->dev,
824 "Failed to allocate the decoder\n");
825 return PTR_ERR(cxled);
826 }
827 cxld = &cxled->cxld;
828 } else {
829 struct cxl_switch_decoder *cxlsd;
830
831 cxlsd = cxl_switch_decoder_alloc(port, target_count);
832 if (IS_ERR(cxlsd)) {
833 dev_warn(&port->dev,
834 "Failed to allocate the decoder\n");
835 return PTR_ERR(cxlsd);
836 }
837 cxld = &cxlsd->cxld;
838 }
839
840 rc = init_hdm_decoder(port, cxld, target_map, hdm, i, &dpa_base);
841 if (rc) {
842 put_device(&cxld->dev);
843 return rc;
844 }
845 rc = add_hdm_decoder(port, cxld, target_map);
846 if (rc) {
847 dev_warn(&port->dev,
848 "Failed to add decoder to port\n");
849 return rc;
850 }
851 }
852
853 return 0;
854 }
855 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);
856
