1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/seq_file.h>
4 #include <linux/device.h>
5 #include <linux/delay.h>
6
7 #include "cxlmem.h"
8 #include "core.h"
9
10 /**
11 * DOC: cxl core hdm
12 *
13 * Compute Express Link Host Managed Device Memory, starting with the
14 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15 * instances per CXL port and per CXL endpoint. Define common helpers
16 * for enumerating these registers and capabilities.
17 */
18
19 DECLARE_RWSEM(cxl_dpa_rwsem);
20
add_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map)21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
22 int *target_map)
23 {
24 int rc;
25
26 rc = cxl_decoder_add_locked(cxld, target_map);
27 if (rc) {
28 put_device(&cxld->dev);
29 dev_err(&port->dev, "Failed to add decoder\n");
30 return rc;
31 }
32
33 rc = cxl_decoder_autoremove(&port->dev, cxld);
34 if (rc)
35 return rc;
36
37 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
38
39 return 0;
40 }
41
42 /*
43 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
44 * single ported host-bridges need not publish a decoder capability when a
45 * passthrough decode can be assumed, i.e. all transactions that the uport sees
46 * are claimed and passed to the single dport. Disable the range until the first
47 * CXL region is enumerated / activated.
48 */
devm_cxl_add_passthrough_decoder(struct cxl_port * port)49 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
50 {
51 struct cxl_switch_decoder *cxlsd;
52 struct cxl_dport *dport = NULL;
53 int single_port_map[1];
54 unsigned long index;
55
56 cxlsd = cxl_switch_decoder_alloc(port, 1);
57 if (IS_ERR(cxlsd))
58 return PTR_ERR(cxlsd);
59
60 device_lock_assert(&port->dev);
61
62 xa_for_each(&port->dports, index, dport)
63 break;
64 single_port_map[0] = dport->port_id;
65
66 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
67 }
68 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
69
parse_hdm_decoder_caps(struct cxl_hdm * cxlhdm)70 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
71 {
72 u32 hdm_cap;
73
74 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
75 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
76 cxlhdm->target_count =
77 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
78 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
79 cxlhdm->interleave_mask |= GENMASK(11, 8);
80 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
81 cxlhdm->interleave_mask |= GENMASK(14, 12);
82 }
83
map_hdm_decoder_regs(struct cxl_port * port,void __iomem * crb,struct cxl_component_regs * regs)84 static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb,
85 struct cxl_component_regs *regs)
86 {
87 struct cxl_register_map map = {
88 .dev = &port->dev,
89 .resource = port->component_reg_phys,
90 .base = crb,
91 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
92 };
93
94 cxl_probe_component_regs(&port->dev, crb, &map.component_map);
95 if (!map.component_map.hdm_decoder.valid) {
96 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
97 /* unique error code to indicate no HDM decoder capability */
98 return -ENODEV;
99 }
100
101 return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM));
102 }
103
should_emulate_decoders(struct cxl_endpoint_dvsec_info * info)104 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
105 {
106 struct cxl_hdm *cxlhdm;
107 void __iomem *hdm;
108 u32 ctrl;
109 int i;
110
111 if (!info)
112 return false;
113
114 cxlhdm = dev_get_drvdata(&info->port->dev);
115 hdm = cxlhdm->regs.hdm_decoder;
116
117 if (!hdm)
118 return true;
119
120 /*
121 * If HDM decoders are present and the driver is in control of
122 * Mem_Enable skip DVSEC based emulation
123 */
124 if (!info->mem_enabled)
125 return false;
126
127 /*
128 * If any decoders are committed already, there should not be any
129 * emulated DVSEC decoders.
130 */
131 for (i = 0; i < cxlhdm->decoder_count; i++) {
132 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
133 dev_dbg(&info->port->dev,
134 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
135 info->port->id, i,
136 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
137 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
138 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
139 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
140 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
141 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
142 return false;
143 }
144
145 return true;
146 }
147
148 /**
149 * devm_cxl_setup_hdm - map HDM decoder component registers
150 * @port: cxl_port to map
151 * @info: cached DVSEC range register info
152 */
devm_cxl_setup_hdm(struct cxl_port * port,struct cxl_endpoint_dvsec_info * info)153 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
154 struct cxl_endpoint_dvsec_info *info)
155 {
156 struct device *dev = &port->dev;
157 struct cxl_hdm *cxlhdm;
158 void __iomem *crb;
159 int rc;
160
161 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
162 if (!cxlhdm)
163 return ERR_PTR(-ENOMEM);
164 cxlhdm->port = port;
165 dev_set_drvdata(dev, cxlhdm);
166
167 crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);
168 if (!crb && info && info->mem_enabled) {
169 cxlhdm->decoder_count = info->ranges;
170 return cxlhdm;
171 } else if (!crb) {
172 dev_err(dev, "No component registers mapped\n");
173 return ERR_PTR(-ENXIO);
174 }
175
176 rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs);
177 iounmap(crb);
178 if (rc)
179 return ERR_PTR(rc);
180
181 parse_hdm_decoder_caps(cxlhdm);
182 if (cxlhdm->decoder_count == 0) {
183 dev_err(dev, "Spec violation. Caps invalid\n");
184 return ERR_PTR(-ENXIO);
185 }
186
187 /*
188 * Now that the hdm capability is parsed, decide if range
189 * register emulation is needed and fixup cxlhdm accordingly.
190 */
191 if (should_emulate_decoders(info)) {
192 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
193 info->ranges > 1 ? "s" : "");
194 cxlhdm->decoder_count = info->ranges;
195 }
196
197 return cxlhdm;
198 }
199 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
200
__cxl_dpa_debug(struct seq_file * file,struct resource * r,int depth)201 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
202 {
203 unsigned long long start = r->start, end = r->end;
204
205 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
206 r->name);
207 }
208
cxl_dpa_debug(struct seq_file * file,struct cxl_dev_state * cxlds)209 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
210 {
211 struct resource *p1, *p2;
212
213 down_read(&cxl_dpa_rwsem);
214 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
215 __cxl_dpa_debug(file, p1, 0);
216 for (p2 = p1->child; p2; p2 = p2->sibling)
217 __cxl_dpa_debug(file, p2, 1);
218 }
219 up_read(&cxl_dpa_rwsem);
220 }
221 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
222
223 /*
224 * Must be called in a context that synchronizes against this decoder's
225 * port ->remove() callback (like an endpoint decoder sysfs attribute)
226 */
__cxl_dpa_release(struct cxl_endpoint_decoder * cxled)227 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
228 {
229 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
230 struct cxl_port *port = cxled_to_port(cxled);
231 struct cxl_dev_state *cxlds = cxlmd->cxlds;
232 struct resource *res = cxled->dpa_res;
233 resource_size_t skip_start;
234
235 lockdep_assert_held_write(&cxl_dpa_rwsem);
236
237 /* save @skip_start, before @res is released */
238 skip_start = res->start - cxled->skip;
239 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
240 if (cxled->skip)
241 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
242 cxled->skip = 0;
243 cxled->dpa_res = NULL;
244 put_device(&cxled->cxld.dev);
245 port->hdm_end--;
246 }
247
cxl_dpa_release(void * cxled)248 static void cxl_dpa_release(void *cxled)
249 {
250 down_write(&cxl_dpa_rwsem);
251 __cxl_dpa_release(cxled);
252 up_write(&cxl_dpa_rwsem);
253 }
254
255 /*
256 * Must be called from context that will not race port device
257 * unregistration, like decoder sysfs attribute methods
258 */
devm_cxl_dpa_release(struct cxl_endpoint_decoder * cxled)259 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
260 {
261 struct cxl_port *port = cxled_to_port(cxled);
262
263 lockdep_assert_held_write(&cxl_dpa_rwsem);
264 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
265 __cxl_dpa_release(cxled);
266 }
267
__cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)268 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
269 resource_size_t base, resource_size_t len,
270 resource_size_t skipped)
271 {
272 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
273 struct cxl_port *port = cxled_to_port(cxled);
274 struct cxl_dev_state *cxlds = cxlmd->cxlds;
275 struct device *dev = &port->dev;
276 struct resource *res;
277
278 lockdep_assert_held_write(&cxl_dpa_rwsem);
279
280 if (!len) {
281 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
282 port->id, cxled->cxld.id);
283 return -EINVAL;
284 }
285
286 if (cxled->dpa_res) {
287 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
288 port->id, cxled->cxld.id, cxled->dpa_res);
289 return -EBUSY;
290 }
291
292 if (port->hdm_end + 1 != cxled->cxld.id) {
293 /*
294 * Assumes alloc and commit order is always in hardware instance
295 * order per expectations from 8.2.5.12.20 Committing Decoder
296 * Programming that enforce decoder[m] committed before
297 * decoder[m+1] commit start.
298 */
299 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
300 cxled->cxld.id, port->id, port->hdm_end + 1);
301 return -EBUSY;
302 }
303
304 if (skipped) {
305 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
306 dev_name(&cxled->cxld.dev), 0);
307 if (!res) {
308 dev_dbg(dev,
309 "decoder%d.%d: failed to reserve skipped space\n",
310 port->id, cxled->cxld.id);
311 return -EBUSY;
312 }
313 }
314 res = __request_region(&cxlds->dpa_res, base, len,
315 dev_name(&cxled->cxld.dev), 0);
316 if (!res) {
317 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
318 port->id, cxled->cxld.id);
319 if (skipped)
320 __release_region(&cxlds->dpa_res, base - skipped,
321 skipped);
322 return -EBUSY;
323 }
324 cxled->dpa_res = res;
325 cxled->skip = skipped;
326
327 if (resource_contains(&cxlds->pmem_res, res))
328 cxled->mode = CXL_DECODER_PMEM;
329 else if (resource_contains(&cxlds->ram_res, res))
330 cxled->mode = CXL_DECODER_RAM;
331 else {
332 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
333 cxled->cxld.id, cxled->dpa_res);
334 cxled->mode = CXL_DECODER_MIXED;
335 }
336
337 port->hdm_end++;
338 get_device(&cxled->cxld.dev);
339 return 0;
340 }
341
devm_cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)342 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
343 resource_size_t base, resource_size_t len,
344 resource_size_t skipped)
345 {
346 struct cxl_port *port = cxled_to_port(cxled);
347 int rc;
348
349 down_write(&cxl_dpa_rwsem);
350 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
351 up_write(&cxl_dpa_rwsem);
352
353 if (rc)
354 return rc;
355
356 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
357 }
358 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL);
359
cxl_dpa_size(struct cxl_endpoint_decoder * cxled)360 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
361 {
362 resource_size_t size = 0;
363
364 down_read(&cxl_dpa_rwsem);
365 if (cxled->dpa_res)
366 size = resource_size(cxled->dpa_res);
367 up_read(&cxl_dpa_rwsem);
368
369 return size;
370 }
371
cxl_dpa_resource_start(struct cxl_endpoint_decoder * cxled)372 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
373 {
374 resource_size_t base = -1;
375
376 down_read(&cxl_dpa_rwsem);
377 if (cxled->dpa_res)
378 base = cxled->dpa_res->start;
379 up_read(&cxl_dpa_rwsem);
380
381 return base;
382 }
383
cxl_dpa_free(struct cxl_endpoint_decoder * cxled)384 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
385 {
386 struct cxl_port *port = cxled_to_port(cxled);
387 struct device *dev = &cxled->cxld.dev;
388 int rc;
389
390 down_write(&cxl_dpa_rwsem);
391 if (!cxled->dpa_res) {
392 rc = 0;
393 goto out;
394 }
395 if (cxled->cxld.region) {
396 dev_dbg(dev, "decoder assigned to: %s\n",
397 dev_name(&cxled->cxld.region->dev));
398 rc = -EBUSY;
399 goto out;
400 }
401 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
402 dev_dbg(dev, "decoder enabled\n");
403 rc = -EBUSY;
404 goto out;
405 }
406 if (cxled->cxld.id != port->hdm_end) {
407 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
408 port->hdm_end);
409 rc = -EBUSY;
410 goto out;
411 }
412 devm_cxl_dpa_release(cxled);
413 rc = 0;
414 out:
415 up_write(&cxl_dpa_rwsem);
416 return rc;
417 }
418
cxl_dpa_set_mode(struct cxl_endpoint_decoder * cxled,enum cxl_decoder_mode mode)419 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
420 enum cxl_decoder_mode mode)
421 {
422 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
423 struct cxl_dev_state *cxlds = cxlmd->cxlds;
424 struct device *dev = &cxled->cxld.dev;
425 int rc;
426
427 switch (mode) {
428 case CXL_DECODER_RAM:
429 case CXL_DECODER_PMEM:
430 break;
431 default:
432 dev_dbg(dev, "unsupported mode: %d\n", mode);
433 return -EINVAL;
434 }
435
436 down_write(&cxl_dpa_rwsem);
437 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
438 rc = -EBUSY;
439 goto out;
440 }
441
442 /*
443 * Only allow modes that are supported by the current partition
444 * configuration
445 */
446 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
447 dev_dbg(dev, "no available pmem capacity\n");
448 rc = -ENXIO;
449 goto out;
450 }
451 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
452 dev_dbg(dev, "no available ram capacity\n");
453 rc = -ENXIO;
454 goto out;
455 }
456
457 cxled->mode = mode;
458 rc = 0;
459 out:
460 up_write(&cxl_dpa_rwsem);
461
462 return rc;
463 }
464
cxl_dpa_alloc(struct cxl_endpoint_decoder * cxled,unsigned long long size)465 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
466 {
467 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
468 resource_size_t free_ram_start, free_pmem_start;
469 struct cxl_port *port = cxled_to_port(cxled);
470 struct cxl_dev_state *cxlds = cxlmd->cxlds;
471 struct device *dev = &cxled->cxld.dev;
472 resource_size_t start, avail, skip;
473 struct resource *p, *last;
474 int rc;
475
476 down_write(&cxl_dpa_rwsem);
477 if (cxled->cxld.region) {
478 dev_dbg(dev, "decoder attached to %s\n",
479 dev_name(&cxled->cxld.region->dev));
480 rc = -EBUSY;
481 goto out;
482 }
483
484 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
485 dev_dbg(dev, "decoder enabled\n");
486 rc = -EBUSY;
487 goto out;
488 }
489
490 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
491 last = p;
492 if (last)
493 free_ram_start = last->end + 1;
494 else
495 free_ram_start = cxlds->ram_res.start;
496
497 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
498 last = p;
499 if (last)
500 free_pmem_start = last->end + 1;
501 else
502 free_pmem_start = cxlds->pmem_res.start;
503
504 if (cxled->mode == CXL_DECODER_RAM) {
505 start = free_ram_start;
506 avail = cxlds->ram_res.end - start + 1;
507 skip = 0;
508 } else if (cxled->mode == CXL_DECODER_PMEM) {
509 resource_size_t skip_start, skip_end;
510
511 start = free_pmem_start;
512 avail = cxlds->pmem_res.end - start + 1;
513 skip_start = free_ram_start;
514
515 /*
516 * If some pmem is already allocated, then that allocation
517 * already handled the skip.
518 */
519 if (cxlds->pmem_res.child &&
520 skip_start == cxlds->pmem_res.child->start)
521 skip_end = skip_start - 1;
522 else
523 skip_end = start - 1;
524 skip = skip_end - skip_start + 1;
525 } else {
526 dev_dbg(dev, "mode not set\n");
527 rc = -EINVAL;
528 goto out;
529 }
530
531 if (size > avail) {
532 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
533 cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
534 &avail);
535 rc = -ENOSPC;
536 goto out;
537 }
538
539 rc = __cxl_dpa_reserve(cxled, start, size, skip);
540 out:
541 up_write(&cxl_dpa_rwsem);
542
543 if (rc)
544 return rc;
545
546 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
547 }
548
cxld_set_interleave(struct cxl_decoder * cxld,u32 * ctrl)549 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
550 {
551 u16 eig;
552 u8 eiw;
553
554 /*
555 * Input validation ensures these warns never fire, but otherwise
556 * suppress unititalized variable usage warnings.
557 */
558 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
559 "invalid interleave_ways: %d\n", cxld->interleave_ways))
560 return;
561 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
562 "invalid interleave_granularity: %d\n",
563 cxld->interleave_granularity))
564 return;
565
566 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
567 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
568 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
569 }
570
cxld_set_type(struct cxl_decoder * cxld,u32 * ctrl)571 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
572 {
573 u32p_replace_bits(ctrl,
574 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
575 CXL_HDM_DECODER0_CTRL_HOSTONLY);
576 }
577
cxlsd_set_targets(struct cxl_switch_decoder * cxlsd,u64 * tgt)578 static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
579 {
580 struct cxl_dport **t = &cxlsd->target[0];
581 int ways = cxlsd->cxld.interleave_ways;
582
583 if (dev_WARN_ONCE(&cxlsd->cxld.dev,
584 ways > 8 || ways > cxlsd->nr_targets,
585 "ways: %d overflows targets: %d\n", ways,
586 cxlsd->nr_targets))
587 return -ENXIO;
588
589 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
590 if (ways > 1)
591 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
592 if (ways > 2)
593 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
594 if (ways > 3)
595 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
596 if (ways > 4)
597 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
598 if (ways > 5)
599 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
600 if (ways > 6)
601 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
602 if (ways > 7)
603 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
604
605 return 0;
606 }
607
608 /*
609 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
610 * committed or error within 10ms, but just be generous with 20ms to account for
611 * clock skew and other marginal behavior
612 */
613 #define COMMIT_TIMEOUT_MS 20
cxld_await_commit(void __iomem * hdm,int id)614 static int cxld_await_commit(void __iomem *hdm, int id)
615 {
616 u32 ctrl;
617 int i;
618
619 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
620 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
621 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
622 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
623 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
624 return -EIO;
625 }
626 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
627 return 0;
628 fsleep(1000);
629 }
630
631 return -ETIMEDOUT;
632 }
633
cxl_decoder_commit(struct cxl_decoder * cxld)634 static int cxl_decoder_commit(struct cxl_decoder *cxld)
635 {
636 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
637 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
638 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
639 int id = cxld->id, rc;
640 u64 base, size;
641 u32 ctrl;
642
643 if (cxld->flags & CXL_DECODER_F_ENABLE)
644 return 0;
645
646 if (port->commit_end + 1 != id) {
647 dev_dbg(&port->dev,
648 "%s: out of order commit, expected decoder%d.%d\n",
649 dev_name(&cxld->dev), port->id, port->commit_end + 1);
650 return -EBUSY;
651 }
652
653 down_read(&cxl_dpa_rwsem);
654 /* common decoder settings */
655 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
656 cxld_set_interleave(cxld, &ctrl);
657 cxld_set_type(cxld, &ctrl);
658 base = cxld->hpa_range.start;
659 size = range_len(&cxld->hpa_range);
660
661 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
662 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
663 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
664 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
665
666 if (is_switch_decoder(&cxld->dev)) {
667 struct cxl_switch_decoder *cxlsd =
668 to_cxl_switch_decoder(&cxld->dev);
669 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
670 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
671 u64 targets;
672
673 rc = cxlsd_set_targets(cxlsd, &targets);
674 if (rc) {
675 dev_dbg(&port->dev, "%s: target configuration error\n",
676 dev_name(&cxld->dev));
677 goto err;
678 }
679
680 writel(upper_32_bits(targets), tl_hi);
681 writel(lower_32_bits(targets), tl_lo);
682 } else {
683 struct cxl_endpoint_decoder *cxled =
684 to_cxl_endpoint_decoder(&cxld->dev);
685 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
686 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
687
688 writel(upper_32_bits(cxled->skip), sk_hi);
689 writel(lower_32_bits(cxled->skip), sk_lo);
690 }
691
692 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
693 up_read(&cxl_dpa_rwsem);
694
695 port->commit_end++;
696 rc = cxld_await_commit(hdm, cxld->id);
697 err:
698 if (rc) {
699 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
700 dev_name(&cxld->dev), rc);
701 cxld->reset(cxld);
702 return rc;
703 }
704 cxld->flags |= CXL_DECODER_F_ENABLE;
705
706 return 0;
707 }
708
cxl_decoder_reset(struct cxl_decoder * cxld)709 static int cxl_decoder_reset(struct cxl_decoder *cxld)
710 {
711 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
712 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
713 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
714 int id = cxld->id;
715 u32 ctrl;
716
717 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
718 return 0;
719
720 if (port->commit_end != id) {
721 dev_dbg(&port->dev,
722 "%s: out of order reset, expected decoder%d.%d\n",
723 dev_name(&cxld->dev), port->id, port->commit_end);
724 return -EBUSY;
725 }
726
727 down_read(&cxl_dpa_rwsem);
728 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
729 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
730 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
731
732 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
733 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
734 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
735 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
736 up_read(&cxl_dpa_rwsem);
737
738 port->commit_end--;
739 cxld->flags &= ~CXL_DECODER_F_ENABLE;
740
741 /* Userspace is now responsible for reconfiguring this decoder */
742 if (is_endpoint_decoder(&cxld->dev)) {
743 struct cxl_endpoint_decoder *cxled;
744
745 cxled = to_cxl_endpoint_decoder(&cxld->dev);
746 cxled->state = CXL_DECODER_STATE_MANUAL;
747 }
748
749 return 0;
750 }
751
cxl_setup_hdm_decoder_from_dvsec(struct cxl_port * port,struct cxl_decoder * cxld,u64 * dpa_base,int which,struct cxl_endpoint_dvsec_info * info)752 static int cxl_setup_hdm_decoder_from_dvsec(
753 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
754 int which, struct cxl_endpoint_dvsec_info *info)
755 {
756 struct cxl_endpoint_decoder *cxled;
757 u64 len;
758 int rc;
759
760 if (!is_cxl_endpoint(port))
761 return -EOPNOTSUPP;
762
763 cxled = to_cxl_endpoint_decoder(&cxld->dev);
764 len = range_len(&info->dvsec_range[which]);
765 if (!len)
766 return -ENOENT;
767
768 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
769 cxld->commit = NULL;
770 cxld->reset = NULL;
771 cxld->hpa_range = info->dvsec_range[which];
772
773 /*
774 * Set the emulated decoder as locked pending additional support to
775 * change the range registers at run time.
776 */
777 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
778 port->commit_end = cxld->id;
779
780 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
781 if (rc) {
782 dev_err(&port->dev,
783 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
784 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
785 return rc;
786 }
787 *dpa_base += len;
788 cxled->state = CXL_DECODER_STATE_AUTO;
789
790 return 0;
791 }
792
init_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map,void __iomem * hdm,int which,u64 * dpa_base,struct cxl_endpoint_dvsec_info * info)793 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
794 int *target_map, void __iomem *hdm, int which,
795 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
796 {
797 struct cxl_endpoint_decoder *cxled = NULL;
798 u64 size, base, skip, dpa_size, lo, hi;
799 bool committed;
800 u32 remainder;
801 int i, rc;
802 u32 ctrl;
803 union {
804 u64 value;
805 unsigned char target_id[8];
806 } target_list;
807
808 if (should_emulate_decoders(info))
809 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
810 which, info);
811
812 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
813 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
814 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
815 base = (hi << 32) + lo;
816 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
817 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
818 size = (hi << 32) + lo;
819 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
820 cxld->commit = cxl_decoder_commit;
821 cxld->reset = cxl_decoder_reset;
822
823 if (!committed)
824 size = 0;
825 if (base == U64_MAX || size == U64_MAX) {
826 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
827 port->id, cxld->id);
828 return -ENXIO;
829 }
830
831 if (info)
832 cxled = to_cxl_endpoint_decoder(&cxld->dev);
833 cxld->hpa_range = (struct range) {
834 .start = base,
835 .end = base + size - 1,
836 };
837
838 /* decoders are enabled if committed */
839 if (committed) {
840 cxld->flags |= CXL_DECODER_F_ENABLE;
841 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
842 cxld->flags |= CXL_DECODER_F_LOCK;
843 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
844 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
845 else
846 cxld->target_type = CXL_DECODER_DEVMEM;
847 if (cxld->id != port->commit_end + 1) {
848 dev_warn(&port->dev,
849 "decoder%d.%d: Committed out of order\n",
850 port->id, cxld->id);
851 return -ENXIO;
852 }
853
854 if (size == 0) {
855 dev_warn(&port->dev,
856 "decoder%d.%d: Committed with zero size\n",
857 port->id, cxld->id);
858 return -ENXIO;
859 }
860 port->commit_end = cxld->id;
861 } else {
862 if (cxled) {
863 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
864 struct cxl_dev_state *cxlds = cxlmd->cxlds;
865
866 /*
867 * Default by devtype until a device arrives that needs
868 * more precision.
869 */
870 if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
871 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
872 else
873 cxld->target_type = CXL_DECODER_DEVMEM;
874 } else {
875 /* To be overridden by region type at commit time */
876 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
877 }
878
879 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
880 cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
881 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
882 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
883 }
884 }
885 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
886 &cxld->interleave_ways);
887 if (rc) {
888 dev_warn(&port->dev,
889 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
890 port->id, cxld->id, ctrl);
891 return rc;
892 }
893 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
894 &cxld->interleave_granularity);
895 if (rc)
896 return rc;
897
898 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
899 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
900 cxld->interleave_ways, cxld->interleave_granularity);
901
902 if (!cxled) {
903 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
904 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
905 target_list.value = (hi << 32) + lo;
906 for (i = 0; i < cxld->interleave_ways; i++)
907 target_map[i] = target_list.target_id[i];
908
909 return 0;
910 }
911
912 if (!committed)
913 return 0;
914
915 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
916 if (remainder) {
917 dev_err(&port->dev,
918 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
919 port->id, cxld->id, size, cxld->interleave_ways);
920 return -ENXIO;
921 }
922 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
923 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
924 skip = (hi << 32) + lo;
925 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
926 if (rc) {
927 dev_err(&port->dev,
928 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
929 port->id, cxld->id, *dpa_base,
930 *dpa_base + dpa_size + skip - 1, rc);
931 return rc;
932 }
933 *dpa_base += dpa_size + skip;
934
935 cxled->state = CXL_DECODER_STATE_AUTO;
936
937 return 0;
938 }
939
cxl_settle_decoders(struct cxl_hdm * cxlhdm)940 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
941 {
942 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
943 int committed, i;
944 u32 ctrl;
945
946 if (!hdm)
947 return;
948
949 /*
950 * Since the register resource was recently claimed via request_region()
951 * be careful about trusting the "not-committed" status until the commit
952 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
953 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
954 * host and target.
955 */
956 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
957 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
958 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
959 committed++;
960 }
961
962 /* ensure that future checks of committed can be trusted */
963 if (committed != cxlhdm->decoder_count)
964 msleep(20);
965 }
966
967 /**
968 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
969 * @cxlhdm: Structure to populate with HDM capabilities
970 * @info: cached DVSEC range register info
971 */
devm_cxl_enumerate_decoders(struct cxl_hdm * cxlhdm,struct cxl_endpoint_dvsec_info * info)972 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
973 struct cxl_endpoint_dvsec_info *info)
974 {
975 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
976 struct cxl_port *port = cxlhdm->port;
977 int i;
978 u64 dpa_base = 0;
979
980 cxl_settle_decoders(cxlhdm);
981
982 for (i = 0; i < cxlhdm->decoder_count; i++) {
983 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
984 int rc, target_count = cxlhdm->target_count;
985 struct cxl_decoder *cxld;
986
987 if (is_cxl_endpoint(port)) {
988 struct cxl_endpoint_decoder *cxled;
989
990 cxled = cxl_endpoint_decoder_alloc(port);
991 if (IS_ERR(cxled)) {
992 dev_warn(&port->dev,
993 "Failed to allocate decoder%d.%d\n",
994 port->id, i);
995 return PTR_ERR(cxled);
996 }
997 cxld = &cxled->cxld;
998 } else {
999 struct cxl_switch_decoder *cxlsd;
1000
1001 cxlsd = cxl_switch_decoder_alloc(port, target_count);
1002 if (IS_ERR(cxlsd)) {
1003 dev_warn(&port->dev,
1004 "Failed to allocate decoder%d.%d\n",
1005 port->id, i);
1006 return PTR_ERR(cxlsd);
1007 }
1008 cxld = &cxlsd->cxld;
1009 }
1010
1011 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
1012 &dpa_base, info);
1013 if (rc) {
1014 dev_warn(&port->dev,
1015 "Failed to initialize decoder%d.%d\n",
1016 port->id, i);
1017 put_device(&cxld->dev);
1018 return rc;
1019 }
1020 rc = add_hdm_decoder(port, cxld, target_map);
1021 if (rc) {
1022 dev_warn(&port->dev,
1023 "Failed to add decoder%d.%d\n", port->id, i);
1024 return rc;
1025 }
1026 }
1027
1028 return 0;
1029 }
1030 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);
1031
