1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7 #include "../i915_selftest.h"
8 #include "igt_flush_test.h"
9
10 #include "mock_context.h"
11
12 struct spinner {
13 struct drm_i915_private *i915;
14 struct drm_i915_gem_object *hws;
15 struct drm_i915_gem_object *obj;
16 u32 *batch;
17 void *seqno;
18 };
19
spinner_init(struct spinner * spin,struct drm_i915_private * i915)20 static int spinner_init(struct spinner *spin, struct drm_i915_private *i915)
21 {
22 unsigned int mode;
23 void *vaddr;
24 int err;
25
26 GEM_BUG_ON(INTEL_GEN(i915) < 8);
27
28 memset(spin, 0, sizeof(*spin));
29 spin->i915 = i915;
30
31 spin->hws = i915_gem_object_create_internal(i915, PAGE_SIZE);
32 if (IS_ERR(spin->hws)) {
33 err = PTR_ERR(spin->hws);
34 goto err;
35 }
36
37 spin->obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
38 if (IS_ERR(spin->obj)) {
39 err = PTR_ERR(spin->obj);
40 goto err_hws;
41 }
42
43 i915_gem_object_set_cache_level(spin->hws, I915_CACHE_LLC);
44 vaddr = i915_gem_object_pin_map(spin->hws, I915_MAP_WB);
45 if (IS_ERR(vaddr)) {
46 err = PTR_ERR(vaddr);
47 goto err_obj;
48 }
49 spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
50
51 mode = HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC;
52 vaddr = i915_gem_object_pin_map(spin->obj, mode);
53 if (IS_ERR(vaddr)) {
54 err = PTR_ERR(vaddr);
55 goto err_unpin_hws;
56 }
57 spin->batch = vaddr;
58
59 return 0;
60
61 err_unpin_hws:
62 i915_gem_object_unpin_map(spin->hws);
63 err_obj:
64 i915_gem_object_put(spin->obj);
65 err_hws:
66 i915_gem_object_put(spin->hws);
67 err:
68 return err;
69 }
70
seqno_offset(u64 fence)71 static unsigned int seqno_offset(u64 fence)
72 {
73 return offset_in_page(sizeof(u32) * fence);
74 }
75
hws_address(const struct i915_vma * hws,const struct i915_request * rq)76 static u64 hws_address(const struct i915_vma *hws,
77 const struct i915_request *rq)
78 {
79 return hws->node.start + seqno_offset(rq->fence.context);
80 }
81
emit_recurse_batch(struct spinner * spin,struct i915_request * rq,u32 arbitration_command)82 static int emit_recurse_batch(struct spinner *spin,
83 struct i915_request *rq,
84 u32 arbitration_command)
85 {
86 struct i915_address_space *vm = &rq->gem_context->ppgtt->vm;
87 struct i915_vma *hws, *vma;
88 u32 *batch;
89 int err;
90
91 vma = i915_vma_instance(spin->obj, vm, NULL);
92 if (IS_ERR(vma))
93 return PTR_ERR(vma);
94
95 hws = i915_vma_instance(spin->hws, vm, NULL);
96 if (IS_ERR(hws))
97 return PTR_ERR(hws);
98
99 err = i915_vma_pin(vma, 0, 0, PIN_USER);
100 if (err)
101 return err;
102
103 err = i915_vma_pin(hws, 0, 0, PIN_USER);
104 if (err)
105 goto unpin_vma;
106
107 err = i915_vma_move_to_active(vma, rq, 0);
108 if (err)
109 goto unpin_hws;
110
111 if (!i915_gem_object_has_active_reference(vma->obj)) {
112 i915_gem_object_get(vma->obj);
113 i915_gem_object_set_active_reference(vma->obj);
114 }
115
116 err = i915_vma_move_to_active(hws, rq, 0);
117 if (err)
118 goto unpin_hws;
119
120 if (!i915_gem_object_has_active_reference(hws->obj)) {
121 i915_gem_object_get(hws->obj);
122 i915_gem_object_set_active_reference(hws->obj);
123 }
124
125 batch = spin->batch;
126
127 *batch++ = MI_STORE_DWORD_IMM_GEN4;
128 *batch++ = lower_32_bits(hws_address(hws, rq));
129 *batch++ = upper_32_bits(hws_address(hws, rq));
130 *batch++ = rq->fence.seqno;
131
132 *batch++ = arbitration_command;
133
134 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
135 *batch++ = lower_32_bits(vma->node.start);
136 *batch++ = upper_32_bits(vma->node.start);
137 *batch++ = MI_BATCH_BUFFER_END; /* not reached */
138
139 i915_gem_chipset_flush(spin->i915);
140
141 err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0);
142
143 unpin_hws:
144 i915_vma_unpin(hws);
145 unpin_vma:
146 i915_vma_unpin(vma);
147 return err;
148 }
149
150 static struct i915_request *
spinner_create_request(struct spinner * spin,struct i915_gem_context * ctx,struct intel_engine_cs * engine,u32 arbitration_command)151 spinner_create_request(struct spinner *spin,
152 struct i915_gem_context *ctx,
153 struct intel_engine_cs *engine,
154 u32 arbitration_command)
155 {
156 struct i915_request *rq;
157 int err;
158
159 rq = i915_request_alloc(engine, ctx);
160 if (IS_ERR(rq))
161 return rq;
162
163 err = emit_recurse_batch(spin, rq, arbitration_command);
164 if (err) {
165 i915_request_add(rq);
166 return ERR_PTR(err);
167 }
168
169 return rq;
170 }
171
hws_seqno(const struct spinner * spin,const struct i915_request * rq)172 static u32 hws_seqno(const struct spinner *spin, const struct i915_request *rq)
173 {
174 u32 *seqno = spin->seqno + seqno_offset(rq->fence.context);
175
176 return READ_ONCE(*seqno);
177 }
178
spinner_end(struct spinner * spin)179 static void spinner_end(struct spinner *spin)
180 {
181 *spin->batch = MI_BATCH_BUFFER_END;
182 i915_gem_chipset_flush(spin->i915);
183 }
184
spinner_fini(struct spinner * spin)185 static void spinner_fini(struct spinner *spin)
186 {
187 spinner_end(spin);
188
189 i915_gem_object_unpin_map(spin->obj);
190 i915_gem_object_put(spin->obj);
191
192 i915_gem_object_unpin_map(spin->hws);
193 i915_gem_object_put(spin->hws);
194 }
195
wait_for_spinner(struct spinner * spin,struct i915_request * rq)196 static bool wait_for_spinner(struct spinner *spin, struct i915_request *rq)
197 {
198 if (!wait_event_timeout(rq->execute,
199 READ_ONCE(rq->global_seqno),
200 msecs_to_jiffies(10)))
201 return false;
202
203 return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq),
204 rq->fence.seqno),
205 10) &&
206 wait_for(i915_seqno_passed(hws_seqno(spin, rq),
207 rq->fence.seqno),
208 1000));
209 }
210
live_sanitycheck(void * arg)211 static int live_sanitycheck(void *arg)
212 {
213 struct drm_i915_private *i915 = arg;
214 struct intel_engine_cs *engine;
215 struct i915_gem_context *ctx;
216 enum intel_engine_id id;
217 struct spinner spin;
218 int err = -ENOMEM;
219
220 if (!HAS_LOGICAL_RING_CONTEXTS(i915))
221 return 0;
222
223 mutex_lock(&i915->drm.struct_mutex);
224
225 if (spinner_init(&spin, i915))
226 goto err_unlock;
227
228 ctx = kernel_context(i915);
229 if (!ctx)
230 goto err_spin;
231
232 for_each_engine(engine, i915, id) {
233 struct i915_request *rq;
234
235 rq = spinner_create_request(&spin, ctx, engine, MI_NOOP);
236 if (IS_ERR(rq)) {
237 err = PTR_ERR(rq);
238 goto err_ctx;
239 }
240
241 i915_request_add(rq);
242 if (!wait_for_spinner(&spin, rq)) {
243 GEM_TRACE("spinner failed to start\n");
244 GEM_TRACE_DUMP();
245 i915_gem_set_wedged(i915);
246 err = -EIO;
247 goto err_ctx;
248 }
249
250 spinner_end(&spin);
251 if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
252 err = -EIO;
253 goto err_ctx;
254 }
255 }
256
257 err = 0;
258 err_ctx:
259 kernel_context_close(ctx);
260 err_spin:
261 spinner_fini(&spin);
262 err_unlock:
263 igt_flush_test(i915, I915_WAIT_LOCKED);
264 mutex_unlock(&i915->drm.struct_mutex);
265 return err;
266 }
267
live_preempt(void * arg)268 static int live_preempt(void *arg)
269 {
270 struct drm_i915_private *i915 = arg;
271 struct i915_gem_context *ctx_hi, *ctx_lo;
272 struct spinner spin_hi, spin_lo;
273 struct intel_engine_cs *engine;
274 enum intel_engine_id id;
275 int err = -ENOMEM;
276
277 if (!HAS_LOGICAL_RING_PREEMPTION(i915))
278 return 0;
279
280 mutex_lock(&i915->drm.struct_mutex);
281
282 if (spinner_init(&spin_hi, i915))
283 goto err_unlock;
284
285 if (spinner_init(&spin_lo, i915))
286 goto err_spin_hi;
287
288 ctx_hi = kernel_context(i915);
289 if (!ctx_hi)
290 goto err_spin_lo;
291 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
292
293 ctx_lo = kernel_context(i915);
294 if (!ctx_lo)
295 goto err_ctx_hi;
296 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
297
298 for_each_engine(engine, i915, id) {
299 struct i915_request *rq;
300
301 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
302 MI_ARB_CHECK);
303 if (IS_ERR(rq)) {
304 err = PTR_ERR(rq);
305 goto err_ctx_lo;
306 }
307
308 i915_request_add(rq);
309 if (!wait_for_spinner(&spin_lo, rq)) {
310 GEM_TRACE("lo spinner failed to start\n");
311 GEM_TRACE_DUMP();
312 i915_gem_set_wedged(i915);
313 err = -EIO;
314 goto err_ctx_lo;
315 }
316
317 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
318 MI_ARB_CHECK);
319 if (IS_ERR(rq)) {
320 spinner_end(&spin_lo);
321 err = PTR_ERR(rq);
322 goto err_ctx_lo;
323 }
324
325 i915_request_add(rq);
326 if (!wait_for_spinner(&spin_hi, rq)) {
327 GEM_TRACE("hi spinner failed to start\n");
328 GEM_TRACE_DUMP();
329 i915_gem_set_wedged(i915);
330 err = -EIO;
331 goto err_ctx_lo;
332 }
333
334 spinner_end(&spin_hi);
335 spinner_end(&spin_lo);
336 if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
337 err = -EIO;
338 goto err_ctx_lo;
339 }
340 }
341
342 err = 0;
343 err_ctx_lo:
344 kernel_context_close(ctx_lo);
345 err_ctx_hi:
346 kernel_context_close(ctx_hi);
347 err_spin_lo:
348 spinner_fini(&spin_lo);
349 err_spin_hi:
350 spinner_fini(&spin_hi);
351 err_unlock:
352 igt_flush_test(i915, I915_WAIT_LOCKED);
353 mutex_unlock(&i915->drm.struct_mutex);
354 return err;
355 }
356
live_late_preempt(void * arg)357 static int live_late_preempt(void *arg)
358 {
359 struct drm_i915_private *i915 = arg;
360 struct i915_gem_context *ctx_hi, *ctx_lo;
361 struct spinner spin_hi, spin_lo;
362 struct intel_engine_cs *engine;
363 struct i915_sched_attr attr = {};
364 enum intel_engine_id id;
365 int err = -ENOMEM;
366
367 if (!HAS_LOGICAL_RING_PREEMPTION(i915))
368 return 0;
369
370 mutex_lock(&i915->drm.struct_mutex);
371
372 if (spinner_init(&spin_hi, i915))
373 goto err_unlock;
374
375 if (spinner_init(&spin_lo, i915))
376 goto err_spin_hi;
377
378 ctx_hi = kernel_context(i915);
379 if (!ctx_hi)
380 goto err_spin_lo;
381
382 ctx_lo = kernel_context(i915);
383 if (!ctx_lo)
384 goto err_ctx_hi;
385
386 for_each_engine(engine, i915, id) {
387 struct i915_request *rq;
388
389 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
390 MI_ARB_CHECK);
391 if (IS_ERR(rq)) {
392 err = PTR_ERR(rq);
393 goto err_ctx_lo;
394 }
395
396 i915_request_add(rq);
397 if (!wait_for_spinner(&spin_lo, rq)) {
398 pr_err("First context failed to start\n");
399 goto err_wedged;
400 }
401
402 rq = spinner_create_request(&spin_hi, ctx_hi, engine, MI_NOOP);
403 if (IS_ERR(rq)) {
404 spinner_end(&spin_lo);
405 err = PTR_ERR(rq);
406 goto err_ctx_lo;
407 }
408
409 i915_request_add(rq);
410 if (wait_for_spinner(&spin_hi, rq)) {
411 pr_err("Second context overtook first?\n");
412 goto err_wedged;
413 }
414
415 attr.priority = I915_PRIORITY_MAX;
416 engine->schedule(rq, &attr);
417
418 if (!wait_for_spinner(&spin_hi, rq)) {
419 pr_err("High priority context failed to preempt the low priority context\n");
420 GEM_TRACE_DUMP();
421 goto err_wedged;
422 }
423
424 spinner_end(&spin_hi);
425 spinner_end(&spin_lo);
426 if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
427 err = -EIO;
428 goto err_ctx_lo;
429 }
430 }
431
432 err = 0;
433 err_ctx_lo:
434 kernel_context_close(ctx_lo);
435 err_ctx_hi:
436 kernel_context_close(ctx_hi);
437 err_spin_lo:
438 spinner_fini(&spin_lo);
439 err_spin_hi:
440 spinner_fini(&spin_hi);
441 err_unlock:
442 igt_flush_test(i915, I915_WAIT_LOCKED);
443 mutex_unlock(&i915->drm.struct_mutex);
444 return err;
445
446 err_wedged:
447 spinner_end(&spin_hi);
448 spinner_end(&spin_lo);
449 i915_gem_set_wedged(i915);
450 err = -EIO;
451 goto err_ctx_lo;
452 }
453
live_preempt_hang(void * arg)454 static int live_preempt_hang(void *arg)
455 {
456 struct drm_i915_private *i915 = arg;
457 struct i915_gem_context *ctx_hi, *ctx_lo;
458 struct spinner spin_hi, spin_lo;
459 struct intel_engine_cs *engine;
460 enum intel_engine_id id;
461 int err = -ENOMEM;
462
463 if (!HAS_LOGICAL_RING_PREEMPTION(i915))
464 return 0;
465
466 if (!intel_has_reset_engine(i915))
467 return 0;
468
469 mutex_lock(&i915->drm.struct_mutex);
470
471 if (spinner_init(&spin_hi, i915))
472 goto err_unlock;
473
474 if (spinner_init(&spin_lo, i915))
475 goto err_spin_hi;
476
477 ctx_hi = kernel_context(i915);
478 if (!ctx_hi)
479 goto err_spin_lo;
480 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
481
482 ctx_lo = kernel_context(i915);
483 if (!ctx_lo)
484 goto err_ctx_hi;
485 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
486
487 for_each_engine(engine, i915, id) {
488 struct i915_request *rq;
489
490 if (!intel_engine_has_preemption(engine))
491 continue;
492
493 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
494 MI_ARB_CHECK);
495 if (IS_ERR(rq)) {
496 err = PTR_ERR(rq);
497 goto err_ctx_lo;
498 }
499
500 i915_request_add(rq);
501 if (!wait_for_spinner(&spin_lo, rq)) {
502 GEM_TRACE("lo spinner failed to start\n");
503 GEM_TRACE_DUMP();
504 i915_gem_set_wedged(i915);
505 err = -EIO;
506 goto err_ctx_lo;
507 }
508
509 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
510 MI_ARB_CHECK);
511 if (IS_ERR(rq)) {
512 spinner_end(&spin_lo);
513 err = PTR_ERR(rq);
514 goto err_ctx_lo;
515 }
516
517 init_completion(&engine->execlists.preempt_hang.completion);
518 engine->execlists.preempt_hang.inject_hang = true;
519
520 i915_request_add(rq);
521
522 if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion,
523 HZ / 10)) {
524 pr_err("Preemption did not occur within timeout!");
525 GEM_TRACE_DUMP();
526 i915_gem_set_wedged(i915);
527 err = -EIO;
528 goto err_ctx_lo;
529 }
530
531 set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
532 i915_reset_engine(engine, NULL);
533 clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
534
535 engine->execlists.preempt_hang.inject_hang = false;
536
537 if (!wait_for_spinner(&spin_hi, rq)) {
538 GEM_TRACE("hi spinner failed to start\n");
539 GEM_TRACE_DUMP();
540 i915_gem_set_wedged(i915);
541 err = -EIO;
542 goto err_ctx_lo;
543 }
544
545 spinner_end(&spin_hi);
546 spinner_end(&spin_lo);
547 if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
548 err = -EIO;
549 goto err_ctx_lo;
550 }
551 }
552
553 err = 0;
554 err_ctx_lo:
555 kernel_context_close(ctx_lo);
556 err_ctx_hi:
557 kernel_context_close(ctx_hi);
558 err_spin_lo:
559 spinner_fini(&spin_lo);
560 err_spin_hi:
561 spinner_fini(&spin_hi);
562 err_unlock:
563 igt_flush_test(i915, I915_WAIT_LOCKED);
564 mutex_unlock(&i915->drm.struct_mutex);
565 return err;
566 }
567
intel_execlists_live_selftests(struct drm_i915_private * i915)568 int intel_execlists_live_selftests(struct drm_i915_private *i915)
569 {
570 static const struct i915_subtest tests[] = {
571 SUBTEST(live_sanitycheck),
572 SUBTEST(live_preempt),
573 SUBTEST(live_late_preempt),
574 SUBTEST(live_preempt_hang),
575 };
576
577 if (!HAS_EXECLISTS(i915))
578 return 0;
579
580 if (i915_terminally_wedged(&i915->gpu_error))
581 return 0;
582
583 return i915_subtests(tests, i915);
584 }
585
