1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include "i915_selftest.h"
7
8 #include "gt/intel_context.h"
9 #include "gt/intel_engine_user.h"
10 #include "gt/intel_gt.h"
11 #include "gt/intel_gpu_commands.h"
12 #include "gem/i915_gem_lmem.h"
13
14 #include "selftests/igt_flush_test.h"
15 #include "selftests/mock_drm.h"
16 #include "selftests/i915_random.h"
17 #include "huge_gem_object.h"
18 #include "mock_context.h"
19
20 #define WIDTH 512
21 #define HEIGHT 32
22
23 struct blit_buffer {
24 struct i915_vma *vma;
25 u32 start_val;
26 u32 tiling;
27 };
28
29 struct tiled_blits {
30 struct intel_context *ce;
31 struct blit_buffer buffers[3];
32 struct blit_buffer scratch;
33 struct i915_vma *batch;
34 u64 hole;
35 u32 width;
36 u32 height;
37 };
38
prepare_blit(const struct tiled_blits * t,struct blit_buffer * dst,struct blit_buffer * src,struct drm_i915_gem_object * batch)39 static int prepare_blit(const struct tiled_blits *t,
40 struct blit_buffer *dst,
41 struct blit_buffer *src,
42 struct drm_i915_gem_object *batch)
43 {
44 const int ver = GRAPHICS_VER(to_i915(batch->base.dev));
45 bool use_64b_reloc = ver >= 8;
46 u32 src_pitch, dst_pitch;
47 u32 cmd, *cs;
48
49 cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
50 if (IS_ERR(cs))
51 return PTR_ERR(cs);
52
53 *cs++ = MI_LOAD_REGISTER_IMM(1);
54 *cs++ = i915_mmio_reg_offset(BCS_SWCTRL);
55 cmd = (BCS_SRC_Y | BCS_DST_Y) << 16;
56 if (src->tiling == I915_TILING_Y)
57 cmd |= BCS_SRC_Y;
58 if (dst->tiling == I915_TILING_Y)
59 cmd |= BCS_DST_Y;
60 *cs++ = cmd;
61
62 cmd = MI_FLUSH_DW;
63 if (ver >= 8)
64 cmd++;
65 *cs++ = cmd;
66 *cs++ = 0;
67 *cs++ = 0;
68 *cs++ = 0;
69
70 cmd = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (8 - 2);
71 if (ver >= 8)
72 cmd += 2;
73
74 src_pitch = t->width * 4;
75 if (src->tiling) {
76 cmd |= XY_SRC_COPY_BLT_SRC_TILED;
77 src_pitch /= 4;
78 }
79
80 dst_pitch = t->width * 4;
81 if (dst->tiling) {
82 cmd |= XY_SRC_COPY_BLT_DST_TILED;
83 dst_pitch /= 4;
84 }
85
86 *cs++ = cmd;
87 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | dst_pitch;
88 *cs++ = 0;
89 *cs++ = t->height << 16 | t->width;
90 *cs++ = lower_32_bits(dst->vma->node.start);
91 if (use_64b_reloc)
92 *cs++ = upper_32_bits(dst->vma->node.start);
93 *cs++ = 0;
94 *cs++ = src_pitch;
95 *cs++ = lower_32_bits(src->vma->node.start);
96 if (use_64b_reloc)
97 *cs++ = upper_32_bits(src->vma->node.start);
98
99 *cs++ = MI_BATCH_BUFFER_END;
100
101 i915_gem_object_flush_map(batch);
102 i915_gem_object_unpin_map(batch);
103
104 return 0;
105 }
106
tiled_blits_destroy_buffers(struct tiled_blits * t)107 static void tiled_blits_destroy_buffers(struct tiled_blits *t)
108 {
109 int i;
110
111 for (i = 0; i < ARRAY_SIZE(t->buffers); i++)
112 i915_vma_put(t->buffers[i].vma);
113
114 i915_vma_put(t->scratch.vma);
115 i915_vma_put(t->batch);
116 }
117
118 static struct i915_vma *
__create_vma(struct tiled_blits * t,size_t size,bool lmem)119 __create_vma(struct tiled_blits *t, size_t size, bool lmem)
120 {
121 struct drm_i915_private *i915 = t->ce->vm->i915;
122 struct drm_i915_gem_object *obj;
123 struct i915_vma *vma;
124
125 if (lmem)
126 obj = i915_gem_object_create_lmem(i915, size, 0);
127 else
128 obj = i915_gem_object_create_shmem(i915, size);
129 if (IS_ERR(obj))
130 return ERR_CAST(obj);
131
132 vma = i915_vma_instance(obj, t->ce->vm, NULL);
133 if (IS_ERR(vma))
134 i915_gem_object_put(obj);
135
136 return vma;
137 }
138
create_vma(struct tiled_blits * t,bool lmem)139 static struct i915_vma *create_vma(struct tiled_blits *t, bool lmem)
140 {
141 return __create_vma(t, PAGE_ALIGN(t->width * t->height * 4), lmem);
142 }
143
tiled_blits_create_buffers(struct tiled_blits * t,int width,int height,struct rnd_state * prng)144 static int tiled_blits_create_buffers(struct tiled_blits *t,
145 int width, int height,
146 struct rnd_state *prng)
147 {
148 struct drm_i915_private *i915 = t->ce->engine->i915;
149 int i;
150
151 t->width = width;
152 t->height = height;
153
154 t->batch = __create_vma(t, PAGE_SIZE, false);
155 if (IS_ERR(t->batch))
156 return PTR_ERR(t->batch);
157
158 t->scratch.vma = create_vma(t, false);
159 if (IS_ERR(t->scratch.vma)) {
160 i915_vma_put(t->batch);
161 return PTR_ERR(t->scratch.vma);
162 }
163
164 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) {
165 struct i915_vma *vma;
166
167 vma = create_vma(t, HAS_LMEM(i915) && i % 2);
168 if (IS_ERR(vma)) {
169 tiled_blits_destroy_buffers(t);
170 return PTR_ERR(vma);
171 }
172
173 t->buffers[i].vma = vma;
174 t->buffers[i].tiling =
175 i915_prandom_u32_max_state(I915_TILING_Y + 1, prng);
176 }
177
178 return 0;
179 }
180
fill_scratch(struct tiled_blits * t,u32 * vaddr,u32 val)181 static void fill_scratch(struct tiled_blits *t, u32 *vaddr, u32 val)
182 {
183 int i;
184
185 t->scratch.start_val = val;
186 for (i = 0; i < t->width * t->height; i++)
187 vaddr[i] = val++;
188
189 i915_gem_object_flush_map(t->scratch.vma->obj);
190 }
191
swizzle_bit(unsigned int bit,u64 offset)192 static u64 swizzle_bit(unsigned int bit, u64 offset)
193 {
194 return (offset & BIT_ULL(bit)) >> (bit - 6);
195 }
196
tiled_offset(const struct intel_gt * gt,u64 v,unsigned int stride,unsigned int tiling)197 static u64 tiled_offset(const struct intel_gt *gt,
198 u64 v,
199 unsigned int stride,
200 unsigned int tiling)
201 {
202 unsigned int swizzle;
203 u64 x, y;
204
205 if (tiling == I915_TILING_NONE)
206 return v;
207
208 y = div64_u64_rem(v, stride, &x);
209
210 if (tiling == I915_TILING_X) {
211 v = div64_u64_rem(y, 8, &y) * stride * 8;
212 v += y * 512;
213 v += div64_u64_rem(x, 512, &x) << 12;
214 v += x;
215
216 swizzle = gt->ggtt->bit_6_swizzle_x;
217 } else {
218 const unsigned int ytile_span = 16;
219 const unsigned int ytile_height = 512;
220
221 v = div64_u64_rem(y, 32, &y) * stride * 32;
222 v += y * ytile_span;
223 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
224 v += x;
225
226 swizzle = gt->ggtt->bit_6_swizzle_y;
227 }
228
229 switch (swizzle) {
230 case I915_BIT_6_SWIZZLE_9:
231 v ^= swizzle_bit(9, v);
232 break;
233 case I915_BIT_6_SWIZZLE_9_10:
234 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
235 break;
236 case I915_BIT_6_SWIZZLE_9_11:
237 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
238 break;
239 case I915_BIT_6_SWIZZLE_9_10_11:
240 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
241 break;
242 }
243
244 return v;
245 }
246
repr_tiling(int tiling)247 static const char *repr_tiling(int tiling)
248 {
249 switch (tiling) {
250 case I915_TILING_NONE: return "linear";
251 case I915_TILING_X: return "X";
252 case I915_TILING_Y: return "Y";
253 default: return "unknown";
254 }
255 }
256
verify_buffer(const struct tiled_blits * t,struct blit_buffer * buf,struct rnd_state * prng)257 static int verify_buffer(const struct tiled_blits *t,
258 struct blit_buffer *buf,
259 struct rnd_state *prng)
260 {
261 const u32 *vaddr;
262 int ret = 0;
263 int x, y, p;
264
265 x = i915_prandom_u32_max_state(t->width, prng);
266 y = i915_prandom_u32_max_state(t->height, prng);
267 p = y * t->width + x;
268
269 vaddr = i915_gem_object_pin_map_unlocked(buf->vma->obj, I915_MAP_WC);
270 if (IS_ERR(vaddr))
271 return PTR_ERR(vaddr);
272
273 if (vaddr[0] != buf->start_val) {
274 ret = -EINVAL;
275 } else {
276 u64 v = tiled_offset(buf->vma->vm->gt,
277 p * 4, t->width * 4,
278 buf->tiling);
279
280 if (vaddr[v / sizeof(*vaddr)] != buf->start_val + p)
281 ret = -EINVAL;
282 }
283 if (ret) {
284 pr_err("Invalid %s tiling detected at (%d, %d), start_val %x\n",
285 repr_tiling(buf->tiling),
286 x, y, buf->start_val);
287 igt_hexdump(vaddr, 4096);
288 }
289
290 i915_gem_object_unpin_map(buf->vma->obj);
291 return ret;
292 }
293
move_to_active(struct i915_vma * vma,struct i915_request * rq,unsigned int flags)294 static int move_to_active(struct i915_vma *vma,
295 struct i915_request *rq,
296 unsigned int flags)
297 {
298 int err;
299
300 i915_vma_lock(vma);
301 err = i915_request_await_object(rq, vma->obj, false);
302 if (err == 0)
303 err = i915_vma_move_to_active(vma, rq, flags);
304 i915_vma_unlock(vma);
305
306 return err;
307 }
308
pin_buffer(struct i915_vma * vma,u64 addr)309 static int pin_buffer(struct i915_vma *vma, u64 addr)
310 {
311 int err;
312
313 if (drm_mm_node_allocated(&vma->node) && vma->node.start != addr) {
314 err = i915_vma_unbind(vma);
315 if (err)
316 return err;
317 }
318
319 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED | addr);
320 if (err)
321 return err;
322
323 return 0;
324 }
325
326 static int
tiled_blit(struct tiled_blits * t,struct blit_buffer * dst,u64 dst_addr,struct blit_buffer * src,u64 src_addr)327 tiled_blit(struct tiled_blits *t,
328 struct blit_buffer *dst, u64 dst_addr,
329 struct blit_buffer *src, u64 src_addr)
330 {
331 struct i915_request *rq;
332 int err;
333
334 err = pin_buffer(src->vma, src_addr);
335 if (err) {
336 pr_err("Cannot pin src @ %llx\n", src_addr);
337 return err;
338 }
339
340 err = pin_buffer(dst->vma, dst_addr);
341 if (err) {
342 pr_err("Cannot pin dst @ %llx\n", dst_addr);
343 goto err_src;
344 }
345
346 err = i915_vma_pin(t->batch, 0, 0, PIN_USER | PIN_HIGH);
347 if (err) {
348 pr_err("cannot pin batch\n");
349 goto err_dst;
350 }
351
352 err = prepare_blit(t, dst, src, t->batch->obj);
353 if (err)
354 goto err_bb;
355
356 rq = intel_context_create_request(t->ce);
357 if (IS_ERR(rq)) {
358 err = PTR_ERR(rq);
359 goto err_bb;
360 }
361
362 err = move_to_active(t->batch, rq, 0);
363 if (!err)
364 err = move_to_active(src->vma, rq, 0);
365 if (!err)
366 err = move_to_active(dst->vma, rq, 0);
367 if (!err)
368 err = rq->engine->emit_bb_start(rq,
369 t->batch->node.start,
370 t->batch->node.size,
371 0);
372 i915_request_get(rq);
373 i915_request_add(rq);
374 if (i915_request_wait(rq, 0, HZ / 2) < 0)
375 err = -ETIME;
376 i915_request_put(rq);
377
378 dst->start_val = src->start_val;
379 err_bb:
380 i915_vma_unpin(t->batch);
381 err_dst:
382 i915_vma_unpin(dst->vma);
383 err_src:
384 i915_vma_unpin(src->vma);
385 return err;
386 }
387
388 static struct tiled_blits *
tiled_blits_create(struct intel_engine_cs * engine,struct rnd_state * prng)389 tiled_blits_create(struct intel_engine_cs *engine, struct rnd_state *prng)
390 {
391 struct drm_mm_node hole;
392 struct tiled_blits *t;
393 u64 hole_size;
394 int err;
395
396 t = kzalloc(sizeof(*t), GFP_KERNEL);
397 if (!t)
398 return ERR_PTR(-ENOMEM);
399
400 t->ce = intel_context_create(engine);
401 if (IS_ERR(t->ce)) {
402 err = PTR_ERR(t->ce);
403 goto err_free;
404 }
405
406 hole_size = 2 * PAGE_ALIGN(WIDTH * HEIGHT * 4);
407 hole_size *= 2; /* room to maneuver */
408 hole_size += 2 * I915_GTT_MIN_ALIGNMENT;
409
410 mutex_lock(&t->ce->vm->mutex);
411 memset(&hole, 0, sizeof(hole));
412 err = drm_mm_insert_node_in_range(&t->ce->vm->mm, &hole,
413 hole_size, 0, I915_COLOR_UNEVICTABLE,
414 0, U64_MAX,
415 DRM_MM_INSERT_BEST);
416 if (!err)
417 drm_mm_remove_node(&hole);
418 mutex_unlock(&t->ce->vm->mutex);
419 if (err) {
420 err = -ENODEV;
421 goto err_put;
422 }
423
424 t->hole = hole.start + I915_GTT_MIN_ALIGNMENT;
425 pr_info("Using hole at %llx\n", t->hole);
426
427 err = tiled_blits_create_buffers(t, WIDTH, HEIGHT, prng);
428 if (err)
429 goto err_put;
430
431 return t;
432
433 err_put:
434 intel_context_put(t->ce);
435 err_free:
436 kfree(t);
437 return ERR_PTR(err);
438 }
439
tiled_blits_destroy(struct tiled_blits * t)440 static void tiled_blits_destroy(struct tiled_blits *t)
441 {
442 tiled_blits_destroy_buffers(t);
443
444 intel_context_put(t->ce);
445 kfree(t);
446 }
447
tiled_blits_prepare(struct tiled_blits * t,struct rnd_state * prng)448 static int tiled_blits_prepare(struct tiled_blits *t,
449 struct rnd_state *prng)
450 {
451 u64 offset = PAGE_ALIGN(t->width * t->height * 4);
452 u32 *map;
453 int err;
454 int i;
455
456 map = i915_gem_object_pin_map_unlocked(t->scratch.vma->obj, I915_MAP_WC);
457 if (IS_ERR(map))
458 return PTR_ERR(map);
459
460 /* Use scratch to fill objects */
461 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) {
462 fill_scratch(t, map, prandom_u32_state(prng));
463 GEM_BUG_ON(verify_buffer(t, &t->scratch, prng));
464
465 err = tiled_blit(t,
466 &t->buffers[i], t->hole + offset,
467 &t->scratch, t->hole);
468 if (err == 0)
469 err = verify_buffer(t, &t->buffers[i], prng);
470 if (err) {
471 pr_err("Failed to create buffer %d\n", i);
472 break;
473 }
474 }
475
476 i915_gem_object_unpin_map(t->scratch.vma->obj);
477 return err;
478 }
479
tiled_blits_bounce(struct tiled_blits * t,struct rnd_state * prng)480 static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng)
481 {
482 u64 offset =
483 round_up(t->width * t->height * 4, 2 * I915_GTT_MIN_ALIGNMENT);
484 int err;
485
486 /* We want to check position invariant tiling across GTT eviction */
487
488 err = tiled_blit(t,
489 &t->buffers[1], t->hole + offset / 2,
490 &t->buffers[0], t->hole + 2 * offset);
491 if (err)
492 return err;
493
494 /* Reposition so that we overlap the old addresses, and slightly off */
495 err = tiled_blit(t,
496 &t->buffers[2], t->hole + I915_GTT_MIN_ALIGNMENT,
497 &t->buffers[1], t->hole + 3 * offset / 2);
498 if (err)
499 return err;
500
501 err = verify_buffer(t, &t->buffers[2], prng);
502 if (err)
503 return err;
504
505 return 0;
506 }
507
__igt_client_tiled_blits(struct intel_engine_cs * engine,struct rnd_state * prng)508 static int __igt_client_tiled_blits(struct intel_engine_cs *engine,
509 struct rnd_state *prng)
510 {
511 struct tiled_blits *t;
512 int err;
513
514 t = tiled_blits_create(engine, prng);
515 if (IS_ERR(t))
516 return PTR_ERR(t);
517
518 err = tiled_blits_prepare(t, prng);
519 if (err)
520 goto out;
521
522 err = tiled_blits_bounce(t, prng);
523 if (err)
524 goto out;
525
526 out:
527 tiled_blits_destroy(t);
528 return err;
529 }
530
has_bit17_swizzle(int sw)531 static bool has_bit17_swizzle(int sw)
532 {
533 return (sw == I915_BIT_6_SWIZZLE_9_10_17 ||
534 sw == I915_BIT_6_SWIZZLE_9_17);
535 }
536
bad_swizzling(struct drm_i915_private * i915)537 static bool bad_swizzling(struct drm_i915_private *i915)
538 {
539 struct i915_ggtt *ggtt = &i915->ggtt;
540
541 if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
542 return true;
543
544 if (has_bit17_swizzle(ggtt->bit_6_swizzle_x) ||
545 has_bit17_swizzle(ggtt->bit_6_swizzle_y))
546 return true;
547
548 return false;
549 }
550
igt_client_tiled_blits(void * arg)551 static int igt_client_tiled_blits(void *arg)
552 {
553 struct drm_i915_private *i915 = arg;
554 I915_RND_STATE(prng);
555 int inst = 0;
556
557 /* Test requires explicit BLT tiling controls */
558 if (GRAPHICS_VER(i915) < 4)
559 return 0;
560
561 if (bad_swizzling(i915)) /* Requires sane (sub-page) swizzling */
562 return 0;
563
564 do {
565 struct intel_engine_cs *engine;
566 int err;
567
568 engine = intel_engine_lookup_user(i915,
569 I915_ENGINE_CLASS_COPY,
570 inst++);
571 if (!engine)
572 return 0;
573
574 err = __igt_client_tiled_blits(engine, &prng);
575 if (err == -ENODEV)
576 err = 0;
577 if (err)
578 return err;
579 } while (1);
580 }
581
i915_gem_client_blt_live_selftests(struct drm_i915_private * i915)582 int i915_gem_client_blt_live_selftests(struct drm_i915_private *i915)
583 {
584 static const struct i915_subtest tests[] = {
585 SUBTEST(igt_client_tiled_blits),
586 };
587
588 if (intel_gt_is_wedged(&i915->gt))
589 return 0;
590
591 return i915_live_subtests(tests, i915);
592 }
593
