1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "i915_drv.h"
7
8 #include "intel_gt_mcr.h"
9 #include "intel_gt_print.h"
10 #include "intel_gt_regs.h"
11
12 /**
13 * DOC: GT Multicast/Replicated (MCR) Register Support
14 *
15 * Some GT registers are designed as "multicast" or "replicated" registers:
16 * multiple instances of the same register share a single MMIO offset. MCR
17 * registers are generally used when the hardware needs to potentially track
18 * independent values of a register per hardware unit (e.g., per-subslice,
19 * per-L3bank, etc.). The specific types of replication that exist vary
20 * per-platform.
21 *
22 * MMIO accesses to MCR registers are controlled according to the settings
23 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
24 * registers can be done in either a (i.e., a single write updates all
25 * instances of the register to the same value) or unicast (a write updates only
26 * one specific instance). Reads of MCR registers always operate in a unicast
27 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.
28 * Selection of a specific MCR instance for unicast operations is referred to
29 * as "steering."
30 *
31 * If MCR register operations are steered toward a hardware unit that is
32 * fused off or currently powered down due to power gating, the MMIO operation
33 * is "terminated" by the hardware. Terminated read operations will return a
34 * value of zero and terminated unicast write operations will be silently
35 * ignored.
36 */
37
38 #define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering)
39
40 static const char * const intel_steering_types[] = {
41 "L3BANK",
42 "MSLICE",
43 "LNCF",
44 "GAM",
45 "DSS",
46 "OADDRM",
47 "INSTANCE 0",
48 };
49
50 static const struct intel_mmio_range icl_l3bank_steering_table[] = {
51 { 0x00B100, 0x00B3FF },
52 {},
53 };
54
55 /*
56 * Although the bspec lists more "MSLICE" ranges than shown here, some of those
57 * are of a "GAM" subclass that has special rules. Thus we use a separate
58 * GAM table farther down for those.
59 */
60 static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = {
61 { 0x00DD00, 0x00DDFF },
62 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
63 {},
64 };
65
66 static const struct intel_mmio_range xehpsdv_gam_steering_table[] = {
67 { 0x004000, 0x004AFF },
68 { 0x00C800, 0x00CFFF },
69 {},
70 };
71
72 static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = {
73 { 0x00B000, 0x00B0FF },
74 { 0x00D800, 0x00D8FF },
75 {},
76 };
77
78 static const struct intel_mmio_range dg2_lncf_steering_table[] = {
79 { 0x00B000, 0x00B0FF },
80 { 0x00D880, 0x00D8FF },
81 {},
82 };
83
84 /*
85 * We have several types of MCR registers on PVC where steering to (0,0)
86 * will always provide us with a non-terminated value. We'll stick them
87 * all in the same table for simplicity.
88 */
89 static const struct intel_mmio_range pvc_instance0_steering_table[] = {
90 { 0x004000, 0x004AFF }, /* HALF-BSLICE */
91 { 0x008800, 0x00887F }, /* CC */
92 { 0x008A80, 0x008AFF }, /* TILEPSMI */
93 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */
94 { 0x00B100, 0x00B3FF }, /* L3BANK */
95 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */
96 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */
97 { 0x00DD00, 0x00DDFF }, /* BSLICE */
98 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */
99 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */
100 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */
101 { 0x024180, 0x0241FF }, /* HALF-BSLICE */
102 {},
103 };
104
105 static const struct intel_mmio_range xelpg_instance0_steering_table[] = {
106 { 0x000B00, 0x000BFF }, /* SQIDI */
107 { 0x001000, 0x001FFF }, /* SQIDI */
108 { 0x004000, 0x0048FF }, /* GAM */
109 { 0x008700, 0x0087FF }, /* SQIDI */
110 { 0x00B000, 0x00B0FF }, /* NODE */
111 { 0x00C800, 0x00CFFF }, /* GAM */
112 { 0x00D880, 0x00D8FF }, /* NODE */
113 { 0x00DD00, 0x00DDFF }, /* OAAL2 */
114 {},
115 };
116
117 static const struct intel_mmio_range xelpg_l3bank_steering_table[] = {
118 { 0x00B100, 0x00B3FF },
119 {},
120 };
121
122 /* DSS steering is used for SLICE ranges as well */
123 static const struct intel_mmio_range xelpg_dss_steering_table[] = {
124 { 0x005200, 0x0052FF }, /* SLICE */
125 { 0x005500, 0x007FFF }, /* SLICE */
126 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
127 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
128 { 0x009680, 0x0096FF }, /* DSS */
129 { 0x00D800, 0x00D87F }, /* SLICE */
130 { 0x00DC00, 0x00DCFF }, /* SLICE */
131 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
132 {},
133 };
134
135 static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = {
136 { 0x393200, 0x39323F },
137 { 0x393400, 0x3934FF },
138 {},
139 };
140
intel_gt_mcr_init(struct intel_gt * gt)141 void intel_gt_mcr_init(struct intel_gt *gt)
142 {
143 struct drm_i915_private *i915 = gt->i915;
144 unsigned long fuse;
145 int i;
146
147 spin_lock_init(>->mcr_lock);
148
149 /*
150 * An mslice is unavailable only if both the meml3 for the slice is
151 * disabled *and* all of the DSS in the slice (quadrant) are disabled.
152 */
153 if (HAS_MSLICE_STEERING(i915)) {
154 gt->info.mslice_mask =
155 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask,
156 GEN_DSS_PER_MSLICE);
157 gt->info.mslice_mask |=
158 (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
159 GEN12_MEML3_EN_MASK);
160
161 if (!gt->info.mslice_mask) /* should be impossible! */
162 gt_warn(gt, "mslice mask all zero!\n");
163 }
164
165 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {
166 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table;
167 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
168 /* Wa_14016747170 */
169 if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
170 IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0))
171 fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK,
172 intel_uncore_read(gt->uncore,
173 MTL_GT_ACTIVITY_FACTOR));
174 else
175 fuse = REG_FIELD_GET(GT_L3_EXC_MASK,
176 intel_uncore_read(gt->uncore, XEHP_FUSE4));
177
178 /*
179 * Despite the register field being named "exclude mask" the
180 * bits actually represent enabled banks (two banks per bit).
181 */
182 for_each_set_bit(i, &fuse, 3)
183 gt->info.l3bank_mask |= 0x3 << 2 * i;
184
185 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table;
186 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table;
187 gt->steering_table[DSS] = xelpg_dss_steering_table;
188 } else if (IS_PONTEVECCHIO(i915)) {
189 gt->steering_table[INSTANCE0] = pvc_instance0_steering_table;
190 } else if (IS_DG2(i915)) {
191 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
192 gt->steering_table[LNCF] = dg2_lncf_steering_table;
193 /*
194 * No need to hook up the GAM table since it has a dedicated
195 * steering control register on DG2 and can use implicit
196 * steering.
197 */
198 } else if (IS_XEHPSDV(i915)) {
199 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
200 gt->steering_table[LNCF] = xehpsdv_lncf_steering_table;
201 gt->steering_table[GAM] = xehpsdv_gam_steering_table;
202 } else if (GRAPHICS_VER(i915) >= 11 &&
203 GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) {
204 gt->steering_table[L3BANK] = icl_l3bank_steering_table;
205 gt->info.l3bank_mask =
206 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
207 GEN10_L3BANK_MASK;
208 if (!gt->info.l3bank_mask) /* should be impossible! */
209 gt_warn(gt, "L3 bank mask is all zero!\n");
210 } else if (GRAPHICS_VER(i915) >= 11) {
211 /*
212 * We expect all modern platforms to have at least some
213 * type of steering that needs to be initialized.
214 */
215 MISSING_CASE(INTEL_INFO(i915)->platform);
216 }
217 }
218
219 /*
220 * Although the rest of the driver should use MCR-specific functions to
221 * read/write MCR registers, we still use the regular intel_uncore_* functions
222 * internally to implement those, so we need a way for the functions in this
223 * file to "cast" an i915_mcr_reg_t into an i915_reg_t.
224 */
mcr_reg_cast(const i915_mcr_reg_t mcr)225 static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr)
226 {
227 i915_reg_t r = { .reg = mcr.reg };
228
229 return r;
230 }
231
232 /*
233 * rw_with_mcr_steering_fw - Access a register with specific MCR steering
234 * @gt: GT to read register from
235 * @reg: register being accessed
236 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access
237 * @group: group number (documented as "sliceid" on older platforms)
238 * @instance: instance number (documented as "subsliceid" on older platforms)
239 * @value: register value to be written (ignored for read)
240 *
241 * Context: The caller must hold the MCR lock
242 * Return: 0 for write access. register value for read access.
243 *
244 * Caller needs to make sure the relevant forcewake wells are up.
245 */
rw_with_mcr_steering_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)246 static u32 rw_with_mcr_steering_fw(struct intel_gt *gt,
247 i915_mcr_reg_t reg, u8 rw_flag,
248 int group, int instance, u32 value)
249 {
250 struct intel_uncore *uncore = gt->uncore;
251 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0;
252
253 lockdep_assert_held(>->mcr_lock);
254
255 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) {
256 /*
257 * Always leave the hardware in multicast mode when doing reads
258 * (see comment about Wa_22013088509 below) and only change it
259 * to unicast mode when doing writes of a specific instance.
260 *
261 * No need to save old steering reg value.
262 */
263 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR,
264 REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
265 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) |
266 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0));
267 } else if (GRAPHICS_VER(uncore->i915) >= 11) {
268 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
269 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance);
270
271 /*
272 * Wa_22013088509
273 *
274 * The setting of the multicast/unicast bit usually wouldn't
275 * matter for read operations (which always return the value
276 * from a single register instance regardless of how that bit
277 * is set), but some platforms have a workaround requiring us
278 * to remain in multicast mode for reads. There's no real
279 * downside to this, so we'll just go ahead and do so on all
280 * platforms; we'll only clear the multicast bit from the mask
281 * when exlicitly doing a write operation.
282 */
283 if (rw_flag == FW_REG_WRITE)
284 mcr_mask |= GEN11_MCR_MULTICAST;
285
286 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
287 old_mcr = mcr;
288
289 mcr &= ~mcr_mask;
290 mcr |= mcr_ss;
291 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
292 } else {
293 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
294 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance);
295
296 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
297 old_mcr = mcr;
298
299 mcr &= ~mcr_mask;
300 mcr |= mcr_ss;
301 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
302 }
303
304 if (rw_flag == FW_REG_READ)
305 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg));
306 else
307 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value);
308
309 /*
310 * For pre-MTL platforms, we need to restore the old value of the
311 * steering control register to ensure that implicit steering continues
312 * to behave as expected. For MTL and beyond, we need only reinstate
313 * the 'multicast' bit (and only if we did a write that cleared it).
314 */
315 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE)
316 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
317 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70))
318 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr);
319
320 return val;
321 }
322
rw_with_mcr_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)323 static u32 rw_with_mcr_steering(struct intel_gt *gt,
324 i915_mcr_reg_t reg, u8 rw_flag,
325 int group, int instance,
326 u32 value)
327 {
328 struct intel_uncore *uncore = gt->uncore;
329 enum forcewake_domains fw_domains;
330 unsigned long flags;
331 u32 val;
332
333 fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg),
334 rw_flag);
335 fw_domains |= intel_uncore_forcewake_for_reg(uncore,
336 GEN8_MCR_SELECTOR,
337 FW_REG_READ | FW_REG_WRITE);
338
339 intel_gt_mcr_lock(gt, &flags);
340 spin_lock(&uncore->lock);
341 intel_uncore_forcewake_get__locked(uncore, fw_domains);
342
343 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value);
344
345 intel_uncore_forcewake_put__locked(uncore, fw_domains);
346 spin_unlock(&uncore->lock);
347 intel_gt_mcr_unlock(gt, flags);
348
349 return val;
350 }
351
352 /**
353 * intel_gt_mcr_lock - Acquire MCR steering lock
354 * @gt: GT structure
355 * @flags: storage to save IRQ flags to
356 *
357 * Performs locking to protect the steering for the duration of an MCR
358 * operation. On MTL and beyond, a hardware lock will also be taken to
359 * serialize access not only for the driver, but also for external hardware and
360 * firmware agents.
361 *
362 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this
363 * function is called, although it may be acquired after this
364 * function call.
365 */
intel_gt_mcr_lock(struct intel_gt * gt,unsigned long * flags)366 void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)
367 __acquires(>->mcr_lock)
368 {
369 unsigned long __flags;
370 int err = 0;
371
372 lockdep_assert_not_held(>->uncore->lock);
373
374 /*
375 * Starting with MTL, we need to coordinate not only with other
376 * driver threads, but also with hardware/firmware agents. A dedicated
377 * locking register is used.
378 */
379 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
380 /*
381 * The steering control and semaphore registers are inside an
382 * "always on" power domain with respect to RC6. However there
383 * are some issues if higher-level platform sleep states are
384 * entering/exiting at the same time these registers are
385 * accessed. Grabbing GT forcewake and holding it over the
386 * entire lock/steer/unlock cycle ensures that those sleep
387 * states have been fully exited before we access these
388 * registers. This wakeref will be released in the unlock
389 * routine.
390 *
391 * This is expected to become a formally documented/numbered
392 * workaround soon.
393 */
394 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_GT);
395
396 err = wait_for(intel_uncore_read_fw(gt->uncore,
397 MTL_STEER_SEMAPHORE) == 0x1, 100);
398 }
399
400 /*
401 * Even on platforms with a hardware lock, we'll continue to grab
402 * a software spinlock too for lockdep purposes. If the hardware lock
403 * was already acquired, there should never be contention on the
404 * software lock.
405 */
406 spin_lock_irqsave(>->mcr_lock, __flags);
407
408 *flags = __flags;
409
410 /*
411 * In theory we should never fail to acquire the HW semaphore; this
412 * would indicate some hardware/firmware is misbehaving and not
413 * releasing it properly.
414 */
415 if (err == -ETIMEDOUT) {
416 gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out");
417 add_taint_for_CI(gt->i915, TAINT_WARN); /* CI is now unreliable */
418 }
419 }
420
421 /**
422 * intel_gt_mcr_unlock - Release MCR steering lock
423 * @gt: GT structure
424 * @flags: IRQ flags to restore
425 *
426 * Releases the lock acquired by intel_gt_mcr_lock().
427 *
428 * Context: Releases gt->mcr_lock
429 */
intel_gt_mcr_unlock(struct intel_gt * gt,unsigned long flags)430 void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags)
431 __releases(>->mcr_lock)
432 {
433 spin_unlock_irqrestore(>->mcr_lock, flags);
434
435 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
436 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);
437
438 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_GT);
439 }
440 }
441
442 /**
443 * intel_gt_mcr_read - read a specific instance of an MCR register
444 * @gt: GT structure
445 * @reg: the MCR register to read
446 * @group: the MCR group
447 * @instance: the MCR instance
448 *
449 * Context: Takes and releases gt->mcr_lock
450 *
451 * Returns the value read from an MCR register after steering toward a specific
452 * group/instance.
453 */
intel_gt_mcr_read(struct intel_gt * gt,i915_mcr_reg_t reg,int group,int instance)454 u32 intel_gt_mcr_read(struct intel_gt *gt,
455 i915_mcr_reg_t reg,
456 int group, int instance)
457 {
458 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0);
459 }
460
461 /**
462 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register
463 * @gt: GT structure
464 * @reg: the MCR register to write
465 * @value: value to write
466 * @group: the MCR group
467 * @instance: the MCR instance
468 *
469 * Write an MCR register in unicast mode after steering toward a specific
470 * group/instance.
471 *
472 * Context: Calls a function that takes and releases gt->mcr_lock
473 */
intel_gt_mcr_unicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value,int group,int instance)474 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value,
475 int group, int instance)
476 {
477 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value);
478 }
479
480 /**
481 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register
482 * @gt: GT structure
483 * @reg: the MCR register to write
484 * @value: value to write
485 *
486 * Write an MCR register in multicast mode to update all instances.
487 *
488 * Context: Takes and releases gt->mcr_lock
489 */
intel_gt_mcr_multicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)490 void intel_gt_mcr_multicast_write(struct intel_gt *gt,
491 i915_mcr_reg_t reg, u32 value)
492 {
493 unsigned long flags;
494
495 intel_gt_mcr_lock(gt, &flags);
496
497 /*
498 * Ensure we have multicast behavior, just in case some non-i915 agent
499 * left the hardware in unicast mode.
500 */
501 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
502 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
503
504 intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value);
505
506 intel_gt_mcr_unlock(gt, flags);
507 }
508
509 /**
510 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register
511 * @gt: GT structure
512 * @reg: the MCR register to write
513 * @value: value to write
514 *
515 * Write an MCR register in multicast mode to update all instances. This
516 * function assumes the caller is already holding any necessary forcewake
517 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should
518 * be obtained automatically.
519 *
520 * Context: The caller must hold gt->mcr_lock.
521 */
intel_gt_mcr_multicast_write_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)522 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value)
523 {
524 lockdep_assert_held(>->mcr_lock);
525
526 /*
527 * Ensure we have multicast behavior, just in case some non-i915 agent
528 * left the hardware in unicast mode.
529 */
530 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
531 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
532
533 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value);
534 }
535
536 /**
537 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations
538 * @gt: GT structure
539 * @reg: the MCR register to read and write
540 * @clear: bits to clear during RMW
541 * @set: bits to set during RMW
542 *
543 * Performs a read-modify-write on an MCR register in a multicast manner.
544 * This operation only makes sense on MCR registers where all instances are
545 * expected to have the same value. The read will target any non-terminated
546 * instance and the write will be applied to all instances.
547 *
548 * This function assumes the caller is already holding any necessary forcewake
549 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should
550 * be obtained automatically.
551 *
552 * Context: Calls functions that take and release gt->mcr_lock
553 *
554 * Returns the old (unmodified) value read.
555 */
intel_gt_mcr_multicast_rmw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 clear,u32 set)556 u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg,
557 u32 clear, u32 set)
558 {
559 u32 val = intel_gt_mcr_read_any(gt, reg);
560
561 intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set);
562
563 return val;
564 }
565
566 /*
567 * reg_needs_read_steering - determine whether a register read requires
568 * explicit steering
569 * @gt: GT structure
570 * @reg: the register to check steering requirements for
571 * @type: type of multicast steering to check
572 *
573 * Determines whether @reg needs explicit steering of a specific type for
574 * reads.
575 *
576 * Returns false if @reg does not belong to a register range of the given
577 * steering type, or if the default (subslice-based) steering IDs are suitable
578 * for @type steering too.
579 */
reg_needs_read_steering(struct intel_gt * gt,i915_mcr_reg_t reg,enum intel_steering_type type)580 static bool reg_needs_read_steering(struct intel_gt *gt,
581 i915_mcr_reg_t reg,
582 enum intel_steering_type type)
583 {
584 u32 offset = i915_mmio_reg_offset(reg);
585 const struct intel_mmio_range *entry;
586
587 if (likely(!gt->steering_table[type]))
588 return false;
589
590 if (IS_GSI_REG(offset))
591 offset += gt->uncore->gsi_offset;
592
593 for (entry = gt->steering_table[type]; entry->end; entry++) {
594 if (offset >= entry->start && offset <= entry->end)
595 return true;
596 }
597
598 return false;
599 }
600
601 /*
602 * get_nonterminated_steering - determines valid IDs for a class of MCR steering
603 * @gt: GT structure
604 * @type: multicast register type
605 * @group: Group ID returned
606 * @instance: Instance ID returned
607 *
608 * Determines group and instance values that will steer reads of the specified
609 * MCR class to a non-terminated instance.
610 */
get_nonterminated_steering(struct intel_gt * gt,enum intel_steering_type type,u8 * group,u8 * instance)611 static void get_nonterminated_steering(struct intel_gt *gt,
612 enum intel_steering_type type,
613 u8 *group, u8 *instance)
614 {
615 u32 dss;
616
617 switch (type) {
618 case L3BANK:
619 *group = 0; /* unused */
620 *instance = __ffs(gt->info.l3bank_mask);
621 break;
622 case MSLICE:
623 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
624 *group = __ffs(gt->info.mslice_mask);
625 *instance = 0; /* unused */
626 break;
627 case LNCF:
628 /*
629 * An LNCF is always present if its mslice is present, so we
630 * can safely just steer to LNCF 0 in all cases.
631 */
632 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
633 *group = __ffs(gt->info.mslice_mask) << 1;
634 *instance = 0; /* unused */
635 break;
636 case GAM:
637 *group = IS_DG2(gt->i915) ? 1 : 0;
638 *instance = 0;
639 break;
640 case DSS:
641 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0);
642 *group = dss / GEN_DSS_PER_GSLICE;
643 *instance = dss % GEN_DSS_PER_GSLICE;
644 break;
645 case INSTANCE0:
646 /*
647 * There are a lot of MCR types for which instance (0, 0)
648 * will always provide a non-terminated value.
649 */
650 *group = 0;
651 *instance = 0;
652 break;
653 case OADDRM:
654 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0))
655 *group = 0;
656 else
657 *group = 1;
658 *instance = 0;
659 break;
660 default:
661 MISSING_CASE(type);
662 *group = 0;
663 *instance = 0;
664 }
665 }
666
667 /**
668 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that
669 * will steer a register to a non-terminated instance
670 * @gt: GT structure
671 * @reg: register for which the steering is required
672 * @group: return variable for group steering
673 * @instance: return variable for instance steering
674 *
675 * This function returns a group/instance pair that is guaranteed to work for
676 * read steering of the given register. Note that a value will be returned even
677 * if the register is not replicated and therefore does not actually require
678 * steering.
679 */
intel_gt_mcr_get_nonterminated_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 * group,u8 * instance)680 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt,
681 i915_mcr_reg_t reg,
682 u8 *group, u8 *instance)
683 {
684 int type;
685
686 for (type = 0; type < NUM_STEERING_TYPES; type++) {
687 if (reg_needs_read_steering(gt, reg, type)) {
688 get_nonterminated_steering(gt, type, group, instance);
689 return;
690 }
691 }
692
693 *group = gt->default_steering.groupid;
694 *instance = gt->default_steering.instanceid;
695 }
696
697 /**
698 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register
699 * @gt: GT structure
700 * @reg: register to read
701 *
702 * Reads a GT MCR register. The read will be steered to a non-terminated
703 * instance (i.e., one that isn't fused off or powered down by power gating).
704 * This function assumes the caller is already holding any necessary forcewake
705 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be
706 * obtained automatically.
707 *
708 * Context: The caller must hold gt->mcr_lock.
709 *
710 * Returns the value from a non-terminated instance of @reg.
711 */
intel_gt_mcr_read_any_fw(struct intel_gt * gt,i915_mcr_reg_t reg)712 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg)
713 {
714 int type;
715 u8 group, instance;
716
717 lockdep_assert_held(>->mcr_lock);
718
719 for (type = 0; type < NUM_STEERING_TYPES; type++) {
720 if (reg_needs_read_steering(gt, reg, type)) {
721 get_nonterminated_steering(gt, type, &group, &instance);
722 return rw_with_mcr_steering_fw(gt, reg,
723 FW_REG_READ,
724 group, instance, 0);
725 }
726 }
727
728 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg));
729 }
730
731 /**
732 * intel_gt_mcr_read_any - reads one instance of an MCR register
733 * @gt: GT structure
734 * @reg: register to read
735 *
736 * Reads a GT MCR register. The read will be steered to a non-terminated
737 * instance (i.e., one that isn't fused off or powered down by power gating).
738 *
739 * Context: Calls a function that takes and releases gt->mcr_lock.
740 *
741 * Returns the value from a non-terminated instance of @reg.
742 */
intel_gt_mcr_read_any(struct intel_gt * gt,i915_mcr_reg_t reg)743 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg)
744 {
745 int type;
746 u8 group, instance;
747
748 for (type = 0; type < NUM_STEERING_TYPES; type++) {
749 if (reg_needs_read_steering(gt, reg, type)) {
750 get_nonterminated_steering(gt, type, &group, &instance);
751 return rw_with_mcr_steering(gt, reg,
752 FW_REG_READ,
753 group, instance, 0);
754 }
755 }
756
757 return intel_uncore_read(gt->uncore, mcr_reg_cast(reg));
758 }
759
report_steering_type(struct drm_printer * p,struct intel_gt * gt,enum intel_steering_type type,bool dump_table)760 static void report_steering_type(struct drm_printer *p,
761 struct intel_gt *gt,
762 enum intel_steering_type type,
763 bool dump_table)
764 {
765 const struct intel_mmio_range *entry;
766 u8 group, instance;
767
768 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);
769
770 if (!gt->steering_table[type]) {
771 drm_printf(p, "%s steering: uses default steering\n",
772 intel_steering_types[type]);
773 return;
774 }
775
776 get_nonterminated_steering(gt, type, &group, &instance);
777 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n",
778 intel_steering_types[type], group, instance);
779
780 if (!dump_table)
781 return;
782
783 for (entry = gt->steering_table[type]; entry->end; entry++)
784 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end);
785 }
786
intel_gt_mcr_report_steering(struct drm_printer * p,struct intel_gt * gt,bool dump_table)787 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
788 bool dump_table)
789 {
790 /*
791 * Starting with MTL we no longer have default steering;
792 * all ranges are explicitly steered.
793 */
794 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70))
795 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n",
796 gt->default_steering.groupid,
797 gt->default_steering.instanceid);
798
799 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
800 for (int i = 0; i < NUM_STEERING_TYPES; i++)
801 if (gt->steering_table[i])
802 report_steering_type(p, gt, i, dump_table);
803 } else if (IS_PONTEVECCHIO(gt->i915)) {
804 report_steering_type(p, gt, INSTANCE0, dump_table);
805 } else if (HAS_MSLICE_STEERING(gt->i915)) {
806 report_steering_type(p, gt, MSLICE, dump_table);
807 report_steering_type(p, gt, LNCF, dump_table);
808 }
809 }
810
811 /**
812 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS
813 * @gt: GT structure
814 * @dss: DSS ID to obtain steering for
815 * @group: pointer to storage for steering group ID
816 * @instance: pointer to storage for steering instance ID
817 *
818 * Returns the steering IDs (via the @group and @instance parameters) that
819 * correspond to a specific subslice/DSS ID.
820 */
intel_gt_mcr_get_ss_steering(struct intel_gt * gt,unsigned int dss,unsigned int * group,unsigned int * instance)821 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,
822 unsigned int *group, unsigned int *instance)
823 {
824 if (IS_PONTEVECCHIO(gt->i915)) {
825 *group = dss / GEN_DSS_PER_CSLICE;
826 *instance = dss % GEN_DSS_PER_CSLICE;
827 } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) {
828 *group = dss / GEN_DSS_PER_GSLICE;
829 *instance = dss % GEN_DSS_PER_GSLICE;
830 } else {
831 *group = dss / GEN_MAX_SS_PER_HSW_SLICE;
832 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE;
833 return;
834 }
835 }
836
837 /**
838 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state
839 * @gt: GT structure
840 * @reg: the register to read
841 * @mask: mask to apply to register value
842 * @value: value to wait for
843 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
844 * @slow_timeout_ms: slow timeout in millisecond
845 *
846 * This routine waits until the target register @reg contains the expected
847 * @value after applying the @mask, i.e. it waits until ::
848 *
849 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value
850 *
851 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
852 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
853 * must be not larger than 20,0000 microseconds.
854 *
855 * This function is basically an MCR-friendly version of
856 * __intel_wait_for_register_fw(). Generally this function will only be used
857 * on GAM registers which are a bit special --- although they're MCR registers,
858 * reads (e.g., waiting for status updates) are always directed to the primary
859 * instance.
860 *
861 * Note that this routine assumes the caller holds forcewake asserted, it is
862 * not suitable for very long waits.
863 *
864 * Context: Calls a function that takes and releases gt->mcr_lock
865 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
866 */
intel_gt_mcr_wait_for_reg(struct intel_gt * gt,i915_mcr_reg_t reg,u32 mask,u32 value,unsigned int fast_timeout_us,unsigned int slow_timeout_ms)867 int intel_gt_mcr_wait_for_reg(struct intel_gt *gt,
868 i915_mcr_reg_t reg,
869 u32 mask,
870 u32 value,
871 unsigned int fast_timeout_us,
872 unsigned int slow_timeout_ms)
873 {
874 int ret;
875
876 lockdep_assert_not_held(>->mcr_lock);
877
878 #define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value)
879
880 /* Catch any overuse of this function */
881 might_sleep_if(slow_timeout_ms);
882 GEM_BUG_ON(fast_timeout_us > 20000);
883 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
884
885 ret = -ETIMEDOUT;
886 if (fast_timeout_us && fast_timeout_us <= 20000)
887 ret = _wait_for_atomic(done, fast_timeout_us, 0);
888 if (ret && slow_timeout_ms)
889 ret = wait_for(done, slow_timeout_ms);
890
891 return ret;
892 #undef done
893 }
894
