1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <linux/list_sort.h>
26 #include <linux/prime_numbers.h>
27
28 #include "gem/i915_gem_context.h"
29 #include "gem/i915_gem_internal.h"
30 #include "gem/i915_gem_region.h"
31 #include "gem/selftests/mock_context.h"
32 #include "gt/intel_context.h"
33 #include "gt/intel_gpu_commands.h"
34 #include "gt/intel_gtt.h"
35
36 #include "i915_random.h"
37 #include "i915_selftest.h"
38 #include "i915_vma_resource.h"
39
40 #include "mock_drm.h"
41 #include "mock_gem_device.h"
42 #include "mock_gtt.h"
43 #include "igt_flush_test.h"
44
cleanup_freed_objects(struct drm_i915_private * i915)45 static void cleanup_freed_objects(struct drm_i915_private *i915)
46 {
47 i915_gem_drain_freed_objects(i915);
48 }
49
fake_free_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)50 static void fake_free_pages(struct drm_i915_gem_object *obj,
51 struct sg_table *pages)
52 {
53 sg_free_table(pages);
54 kfree(pages);
55 }
56
fake_get_pages(struct drm_i915_gem_object * obj)57 static int fake_get_pages(struct drm_i915_gem_object *obj)
58 {
59 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
60 #define PFN_BIAS 0x1000
61 struct sg_table *pages;
62 struct scatterlist *sg;
63 unsigned int sg_page_sizes;
64 typeof(obj->base.size) rem;
65
66 pages = kmalloc(sizeof(*pages), GFP);
67 if (!pages)
68 return -ENOMEM;
69
70 rem = round_up(obj->base.size, BIT(31)) >> 31;
71 if (sg_alloc_table(pages, rem, GFP)) {
72 kfree(pages);
73 return -ENOMEM;
74 }
75
76 sg_page_sizes = 0;
77 rem = obj->base.size;
78 for (sg = pages->sgl; sg; sg = sg_next(sg)) {
79 unsigned long len = min_t(typeof(rem), rem, BIT(31));
80
81 GEM_BUG_ON(!len);
82 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
83 sg_dma_address(sg) = page_to_phys(sg_page(sg));
84 sg_dma_len(sg) = len;
85 sg_page_sizes |= len;
86
87 rem -= len;
88 }
89 GEM_BUG_ON(rem);
90
91 __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
92
93 return 0;
94 #undef GFP
95 }
96
fake_put_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)97 static void fake_put_pages(struct drm_i915_gem_object *obj,
98 struct sg_table *pages)
99 {
100 fake_free_pages(obj, pages);
101 obj->mm.dirty = false;
102 }
103
104 static const struct drm_i915_gem_object_ops fake_ops = {
105 .name = "fake-gem",
106 .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
107 .get_pages = fake_get_pages,
108 .put_pages = fake_put_pages,
109 };
110
111 static struct drm_i915_gem_object *
fake_dma_object(struct drm_i915_private * i915,u64 size)112 fake_dma_object(struct drm_i915_private *i915, u64 size)
113 {
114 static struct lock_class_key lock_class;
115 struct drm_i915_gem_object *obj;
116
117 GEM_BUG_ON(!size);
118 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
119
120 if (overflows_type(size, obj->base.size))
121 return ERR_PTR(-E2BIG);
122
123 obj = i915_gem_object_alloc();
124 if (!obj)
125 goto err;
126
127 drm_gem_private_object_init(&i915->drm, &obj->base, size);
128 i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
129
130 i915_gem_object_set_volatile(obj);
131
132 obj->write_domain = I915_GEM_DOMAIN_CPU;
133 obj->read_domains = I915_GEM_DOMAIN_CPU;
134 obj->cache_level = I915_CACHE_NONE;
135
136 /* Preallocate the "backing storage" */
137 if (i915_gem_object_pin_pages_unlocked(obj))
138 goto err_obj;
139
140 i915_gem_object_unpin_pages(obj);
141 return obj;
142
143 err_obj:
144 i915_gem_object_put(obj);
145 err:
146 return ERR_PTR(-ENOMEM);
147 }
148
igt_ppgtt_alloc(void * arg)149 static int igt_ppgtt_alloc(void *arg)
150 {
151 struct drm_i915_private *dev_priv = arg;
152 struct i915_ppgtt *ppgtt;
153 struct i915_gem_ww_ctx ww;
154 u64 size, last, limit;
155 int err = 0;
156
157 /* Allocate a ppggt and try to fill the entire range */
158
159 if (!HAS_PPGTT(dev_priv))
160 return 0;
161
162 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
163 if (IS_ERR(ppgtt))
164 return PTR_ERR(ppgtt);
165
166 if (!ppgtt->vm.allocate_va_range)
167 goto err_ppgtt_cleanup;
168
169 /*
170 * While we only allocate the page tables here and so we could
171 * address a much larger GTT than we could actually fit into
172 * RAM, a practical limit is the amount of physical pages in the system.
173 * This should ensure that we do not run into the oomkiller during
174 * the test and take down the machine wilfully.
175 */
176 limit = totalram_pages() << PAGE_SHIFT;
177 limit = min(ppgtt->vm.total, limit);
178
179 i915_gem_ww_ctx_init(&ww, false);
180 retry:
181 err = i915_vm_lock_objects(&ppgtt->vm, &ww);
182 if (err)
183 goto err_ppgtt_cleanup;
184
185 /* Check we can allocate the entire range */
186 for (size = 4096; size <= limit; size <<= 2) {
187 struct i915_vm_pt_stash stash = {};
188
189 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
190 if (err)
191 goto err_ppgtt_cleanup;
192
193 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
194 if (err) {
195 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
196 goto err_ppgtt_cleanup;
197 }
198
199 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
200 cond_resched();
201
202 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
203
204 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
205 }
206
207 /* Check we can incrementally allocate the entire range */
208 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
209 struct i915_vm_pt_stash stash = {};
210
211 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
212 if (err)
213 goto err_ppgtt_cleanup;
214
215 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
216 if (err) {
217 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
218 goto err_ppgtt_cleanup;
219 }
220
221 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
222 last, size - last);
223 cond_resched();
224
225 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
226 }
227
228 err_ppgtt_cleanup:
229 if (err == -EDEADLK) {
230 err = i915_gem_ww_ctx_backoff(&ww);
231 if (!err)
232 goto retry;
233 }
234 i915_gem_ww_ctx_fini(&ww);
235
236 i915_vm_put(&ppgtt->vm);
237 return err;
238 }
239
lowlevel_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)240 static int lowlevel_hole(struct i915_address_space *vm,
241 u64 hole_start, u64 hole_end,
242 unsigned long end_time)
243 {
244 const unsigned int min_alignment =
245 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
246 I915_RND_STATE(seed_prng);
247 struct i915_vma_resource *mock_vma_res;
248 unsigned int size;
249
250 mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL);
251 if (!mock_vma_res)
252 return -ENOMEM;
253
254 /* Keep creating larger objects until one cannot fit into the hole */
255 for (size = 12; (hole_end - hole_start) >> size; size++) {
256 I915_RND_SUBSTATE(prng, seed_prng);
257 struct drm_i915_gem_object *obj;
258 unsigned int *order, count, n;
259 u64 hole_size, aligned_size;
260
261 aligned_size = max_t(u32, ilog2(min_alignment), size);
262 hole_size = (hole_end - hole_start) >> aligned_size;
263 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
264 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
265 count = hole_size >> 1;
266 if (!count) {
267 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
268 __func__, hole_start, hole_end, size, hole_size);
269 break;
270 }
271
272 do {
273 order = i915_random_order(count, &prng);
274 if (order)
275 break;
276 } while (count >>= 1);
277 if (!count) {
278 kfree(mock_vma_res);
279 return -ENOMEM;
280 }
281 GEM_BUG_ON(!order);
282
283 GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total);
284 GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end);
285
286 /* Ignore allocation failures (i.e. don't report them as
287 * a test failure) as we are purposefully allocating very
288 * large objects without checking that we have sufficient
289 * memory. We expect to hit -ENOMEM.
290 */
291
292 obj = fake_dma_object(vm->i915, BIT_ULL(size));
293 if (IS_ERR(obj)) {
294 kfree(order);
295 break;
296 }
297
298 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
299
300 if (i915_gem_object_pin_pages_unlocked(obj)) {
301 i915_gem_object_put(obj);
302 kfree(order);
303 break;
304 }
305
306 for (n = 0; n < count; n++) {
307 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
308 intel_wakeref_t wakeref;
309
310 GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total);
311
312 if (igt_timeout(end_time,
313 "%s timed out before %d/%d\n",
314 __func__, n, count)) {
315 hole_end = hole_start; /* quit */
316 break;
317 }
318
319 if (vm->allocate_va_range) {
320 struct i915_vm_pt_stash stash = {};
321 struct i915_gem_ww_ctx ww;
322 int err;
323
324 i915_gem_ww_ctx_init(&ww, false);
325 retry:
326 err = i915_vm_lock_objects(vm, &ww);
327 if (err)
328 goto alloc_vm_end;
329
330 err = -ENOMEM;
331 if (i915_vm_alloc_pt_stash(vm, &stash,
332 BIT_ULL(size)))
333 goto alloc_vm_end;
334
335 err = i915_vm_map_pt_stash(vm, &stash);
336 if (!err)
337 vm->allocate_va_range(vm, &stash,
338 addr, BIT_ULL(size));
339 i915_vm_free_pt_stash(vm, &stash);
340 alloc_vm_end:
341 if (err == -EDEADLK) {
342 err = i915_gem_ww_ctx_backoff(&ww);
343 if (!err)
344 goto retry;
345 }
346 i915_gem_ww_ctx_fini(&ww);
347
348 if (err)
349 break;
350 }
351
352 mock_vma_res->bi.pages = obj->mm.pages;
353 mock_vma_res->node_size = BIT_ULL(aligned_size);
354 mock_vma_res->start = addr;
355
356 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
357 vm->insert_entries(vm, mock_vma_res,
358 I915_CACHE_NONE, 0);
359 }
360 count = n;
361
362 i915_random_reorder(order, count, &prng);
363 for (n = 0; n < count; n++) {
364 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
365 intel_wakeref_t wakeref;
366
367 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
368 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
369 vm->clear_range(vm, addr, BIT_ULL(size));
370 }
371
372 i915_gem_object_unpin_pages(obj);
373 i915_gem_object_put(obj);
374
375 kfree(order);
376
377 cleanup_freed_objects(vm->i915);
378 }
379
380 kfree(mock_vma_res);
381 return 0;
382 }
383
close_object_list(struct list_head * objects,struct i915_address_space * vm)384 static void close_object_list(struct list_head *objects,
385 struct i915_address_space *vm)
386 {
387 struct drm_i915_gem_object *obj, *on;
388 int ignored;
389
390 list_for_each_entry_safe(obj, on, objects, st_link) {
391 struct i915_vma *vma;
392
393 vma = i915_vma_instance(obj, vm, NULL);
394 if (!IS_ERR(vma))
395 ignored = i915_vma_unbind_unlocked(vma);
396
397 list_del(&obj->st_link);
398 i915_gem_object_put(obj);
399 }
400 }
401
fill_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)402 static int fill_hole(struct i915_address_space *vm,
403 u64 hole_start, u64 hole_end,
404 unsigned long end_time)
405 {
406 const u64 hole_size = hole_end - hole_start;
407 struct drm_i915_gem_object *obj;
408 const unsigned int min_alignment =
409 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
410 const unsigned long max_pages =
411 min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment));
412 const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
413 unsigned long npages, prime, flags;
414 struct i915_vma *vma;
415 LIST_HEAD(objects);
416 int err;
417
418 /* Try binding many VMA working inwards from either edge */
419
420 flags = PIN_OFFSET_FIXED | PIN_USER;
421 if (i915_is_ggtt(vm))
422 flags |= PIN_GLOBAL;
423
424 for_each_prime_number_from(prime, 2, max_step) {
425 for (npages = 1; npages <= max_pages; npages *= prime) {
426 const u64 full_size = npages << PAGE_SHIFT;
427 const struct {
428 const char *name;
429 u64 offset;
430 int step;
431 } phases[] = {
432 { "top-down", hole_end, -1, },
433 { "bottom-up", hole_start, 1, },
434 { }
435 }, *p;
436
437 obj = fake_dma_object(vm->i915, full_size);
438 if (IS_ERR(obj))
439 break;
440
441 list_add(&obj->st_link, &objects);
442
443 /* Align differing sized objects against the edges, and
444 * check we don't walk off into the void when binding
445 * them into the GTT.
446 */
447 for (p = phases; p->name; p++) {
448 u64 offset;
449
450 offset = p->offset;
451 list_for_each_entry(obj, &objects, st_link) {
452 u64 aligned_size = round_up(obj->base.size,
453 min_alignment);
454
455 vma = i915_vma_instance(obj, vm, NULL);
456 if (IS_ERR(vma))
457 continue;
458
459 if (p->step < 0) {
460 if (offset < hole_start + aligned_size)
461 break;
462 offset -= aligned_size;
463 }
464
465 err = i915_vma_pin(vma, 0, 0, offset | flags);
466 if (err) {
467 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
468 __func__, p->name, err, npages, prime, offset);
469 goto err;
470 }
471
472 if (!drm_mm_node_allocated(&vma->node) ||
473 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
474 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
475 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
476 offset);
477 err = -EINVAL;
478 goto err;
479 }
480
481 i915_vma_unpin(vma);
482
483 if (p->step > 0) {
484 if (offset + aligned_size > hole_end)
485 break;
486 offset += aligned_size;
487 }
488 }
489
490 offset = p->offset;
491 list_for_each_entry(obj, &objects, st_link) {
492 u64 aligned_size = round_up(obj->base.size,
493 min_alignment);
494
495 vma = i915_vma_instance(obj, vm, NULL);
496 if (IS_ERR(vma))
497 continue;
498
499 if (p->step < 0) {
500 if (offset < hole_start + aligned_size)
501 break;
502 offset -= aligned_size;
503 }
504
505 if (!drm_mm_node_allocated(&vma->node) ||
506 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
507 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
508 __func__, p->name, vma->node.start, vma->node.size,
509 offset);
510 err = -EINVAL;
511 goto err;
512 }
513
514 err = i915_vma_unbind_unlocked(vma);
515 if (err) {
516 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
517 __func__, p->name, vma->node.start, vma->node.size,
518 err);
519 goto err;
520 }
521
522 if (p->step > 0) {
523 if (offset + aligned_size > hole_end)
524 break;
525 offset += aligned_size;
526 }
527 }
528
529 offset = p->offset;
530 list_for_each_entry_reverse(obj, &objects, st_link) {
531 u64 aligned_size = round_up(obj->base.size,
532 min_alignment);
533
534 vma = i915_vma_instance(obj, vm, NULL);
535 if (IS_ERR(vma))
536 continue;
537
538 if (p->step < 0) {
539 if (offset < hole_start + aligned_size)
540 break;
541 offset -= aligned_size;
542 }
543
544 err = i915_vma_pin(vma, 0, 0, offset | flags);
545 if (err) {
546 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
547 __func__, p->name, err, npages, prime, offset);
548 goto err;
549 }
550
551 if (!drm_mm_node_allocated(&vma->node) ||
552 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
553 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
554 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
555 offset);
556 err = -EINVAL;
557 goto err;
558 }
559
560 i915_vma_unpin(vma);
561
562 if (p->step > 0) {
563 if (offset + aligned_size > hole_end)
564 break;
565 offset += aligned_size;
566 }
567 }
568
569 offset = p->offset;
570 list_for_each_entry_reverse(obj, &objects, st_link) {
571 u64 aligned_size = round_up(obj->base.size,
572 min_alignment);
573
574 vma = i915_vma_instance(obj, vm, NULL);
575 if (IS_ERR(vma))
576 continue;
577
578 if (p->step < 0) {
579 if (offset < hole_start + aligned_size)
580 break;
581 offset -= aligned_size;
582 }
583
584 if (!drm_mm_node_allocated(&vma->node) ||
585 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
586 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
587 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
588 offset);
589 err = -EINVAL;
590 goto err;
591 }
592
593 err = i915_vma_unbind_unlocked(vma);
594 if (err) {
595 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
596 __func__, p->name, vma->node.start, vma->node.size,
597 err);
598 goto err;
599 }
600
601 if (p->step > 0) {
602 if (offset + aligned_size > hole_end)
603 break;
604 offset += aligned_size;
605 }
606 }
607 }
608
609 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
610 __func__, npages, prime)) {
611 err = -EINTR;
612 goto err;
613 }
614 }
615
616 close_object_list(&objects, vm);
617 cleanup_freed_objects(vm->i915);
618 }
619
620 return 0;
621
622 err:
623 close_object_list(&objects, vm);
624 return err;
625 }
626
walk_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)627 static int walk_hole(struct i915_address_space *vm,
628 u64 hole_start, u64 hole_end,
629 unsigned long end_time)
630 {
631 const u64 hole_size = hole_end - hole_start;
632 const unsigned long max_pages =
633 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
634 unsigned long min_alignment;
635 unsigned long flags;
636 u64 size;
637
638 /* Try binding a single VMA in different positions within the hole */
639
640 flags = PIN_OFFSET_FIXED | PIN_USER;
641 if (i915_is_ggtt(vm))
642 flags |= PIN_GLOBAL;
643
644 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
645
646 for_each_prime_number_from(size, 1, max_pages) {
647 struct drm_i915_gem_object *obj;
648 struct i915_vma *vma;
649 u64 addr;
650 int err = 0;
651
652 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
653 if (IS_ERR(obj))
654 break;
655
656 vma = i915_vma_instance(obj, vm, NULL);
657 if (IS_ERR(vma)) {
658 err = PTR_ERR(vma);
659 goto err_put;
660 }
661
662 for (addr = hole_start;
663 addr + obj->base.size < hole_end;
664 addr += round_up(obj->base.size, min_alignment)) {
665 err = i915_vma_pin(vma, 0, 0, addr | flags);
666 if (err) {
667 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
668 __func__, addr, vma->size,
669 hole_start, hole_end, err);
670 goto err_put;
671 }
672 i915_vma_unpin(vma);
673
674 if (!drm_mm_node_allocated(&vma->node) ||
675 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
676 pr_err("%s incorrect at %llx + %llx\n",
677 __func__, addr, vma->size);
678 err = -EINVAL;
679 goto err_put;
680 }
681
682 err = i915_vma_unbind_unlocked(vma);
683 if (err) {
684 pr_err("%s unbind failed at %llx + %llx with err=%d\n",
685 __func__, addr, vma->size, err);
686 goto err_put;
687 }
688
689 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
690
691 if (igt_timeout(end_time,
692 "%s timed out at %llx\n",
693 __func__, addr)) {
694 err = -EINTR;
695 goto err_put;
696 }
697 }
698
699 err_put:
700 i915_gem_object_put(obj);
701 if (err)
702 return err;
703
704 cleanup_freed_objects(vm->i915);
705 }
706
707 return 0;
708 }
709
pot_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)710 static int pot_hole(struct i915_address_space *vm,
711 u64 hole_start, u64 hole_end,
712 unsigned long end_time)
713 {
714 struct drm_i915_gem_object *obj;
715 struct i915_vma *vma;
716 unsigned int min_alignment;
717 unsigned long flags;
718 unsigned int pot;
719 int err = 0;
720
721 flags = PIN_OFFSET_FIXED | PIN_USER;
722 if (i915_is_ggtt(vm))
723 flags |= PIN_GLOBAL;
724
725 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
726
727 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
728 if (IS_ERR(obj))
729 return PTR_ERR(obj);
730
731 vma = i915_vma_instance(obj, vm, NULL);
732 if (IS_ERR(vma)) {
733 err = PTR_ERR(vma);
734 goto err_obj;
735 }
736
737 /* Insert a pair of pages across every pot boundary within the hole */
738 for (pot = fls64(hole_end - 1) - 1;
739 pot > ilog2(2 * min_alignment);
740 pot--) {
741 u64 step = BIT_ULL(pot);
742 u64 addr;
743
744 for (addr = round_up(hole_start + min_alignment, step) - min_alignment;
745 hole_end > addr && hole_end - addr >= 2 * min_alignment;
746 addr += step) {
747 err = i915_vma_pin(vma, 0, 0, addr | flags);
748 if (err) {
749 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
750 __func__,
751 addr,
752 hole_start, hole_end,
753 err);
754 goto err_obj;
755 }
756
757 if (!drm_mm_node_allocated(&vma->node) ||
758 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
759 pr_err("%s incorrect at %llx + %llx\n",
760 __func__, addr, vma->size);
761 i915_vma_unpin(vma);
762 err = i915_vma_unbind_unlocked(vma);
763 err = -EINVAL;
764 goto err_obj;
765 }
766
767 i915_vma_unpin(vma);
768 err = i915_vma_unbind_unlocked(vma);
769 GEM_BUG_ON(err);
770 }
771
772 if (igt_timeout(end_time,
773 "%s timed out after %d/%d\n",
774 __func__, pot, fls64(hole_end - 1) - 1)) {
775 err = -EINTR;
776 goto err_obj;
777 }
778 }
779
780 err_obj:
781 i915_gem_object_put(obj);
782 return err;
783 }
784
drunk_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)785 static int drunk_hole(struct i915_address_space *vm,
786 u64 hole_start, u64 hole_end,
787 unsigned long end_time)
788 {
789 I915_RND_STATE(prng);
790 unsigned int min_alignment;
791 unsigned int size;
792 unsigned long flags;
793
794 flags = PIN_OFFSET_FIXED | PIN_USER;
795 if (i915_is_ggtt(vm))
796 flags |= PIN_GLOBAL;
797
798 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
799
800 /* Keep creating larger objects until one cannot fit into the hole */
801 for (size = 12; (hole_end - hole_start) >> size; size++) {
802 struct drm_i915_gem_object *obj;
803 unsigned int *order, count, n;
804 struct i915_vma *vma;
805 u64 hole_size, aligned_size;
806 int err = -ENODEV;
807
808 aligned_size = max_t(u32, ilog2(min_alignment), size);
809 hole_size = (hole_end - hole_start) >> aligned_size;
810 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
811 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
812 count = hole_size >> 1;
813 if (!count) {
814 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
815 __func__, hole_start, hole_end, size, hole_size);
816 break;
817 }
818
819 do {
820 order = i915_random_order(count, &prng);
821 if (order)
822 break;
823 } while (count >>= 1);
824 if (!count)
825 return -ENOMEM;
826 GEM_BUG_ON(!order);
827
828 /* Ignore allocation failures (i.e. don't report them as
829 * a test failure) as we are purposefully allocating very
830 * large objects without checking that we have sufficient
831 * memory. We expect to hit -ENOMEM.
832 */
833
834 obj = fake_dma_object(vm->i915, BIT_ULL(size));
835 if (IS_ERR(obj)) {
836 kfree(order);
837 break;
838 }
839
840 vma = i915_vma_instance(obj, vm, NULL);
841 if (IS_ERR(vma)) {
842 err = PTR_ERR(vma);
843 goto err_obj;
844 }
845
846 GEM_BUG_ON(vma->size != BIT_ULL(size));
847
848 for (n = 0; n < count; n++) {
849 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
850
851 err = i915_vma_pin(vma, 0, 0, addr | flags);
852 if (err) {
853 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
854 __func__,
855 addr, BIT_ULL(size),
856 hole_start, hole_end,
857 err);
858 goto err_obj;
859 }
860
861 if (!drm_mm_node_allocated(&vma->node) ||
862 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
863 pr_err("%s incorrect at %llx + %llx\n",
864 __func__, addr, BIT_ULL(size));
865 i915_vma_unpin(vma);
866 err = i915_vma_unbind_unlocked(vma);
867 err = -EINVAL;
868 goto err_obj;
869 }
870
871 i915_vma_unpin(vma);
872 err = i915_vma_unbind_unlocked(vma);
873 GEM_BUG_ON(err);
874
875 if (igt_timeout(end_time,
876 "%s timed out after %d/%d\n",
877 __func__, n, count)) {
878 err = -EINTR;
879 goto err_obj;
880 }
881 }
882
883 err_obj:
884 i915_gem_object_put(obj);
885 kfree(order);
886 if (err)
887 return err;
888
889 cleanup_freed_objects(vm->i915);
890 }
891
892 return 0;
893 }
894
__shrink_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)895 static int __shrink_hole(struct i915_address_space *vm,
896 u64 hole_start, u64 hole_end,
897 unsigned long end_time)
898 {
899 struct drm_i915_gem_object *obj;
900 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
901 unsigned int min_alignment;
902 unsigned int order = 12;
903 LIST_HEAD(objects);
904 int err = 0;
905 u64 addr;
906
907 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
908
909 /* Keep creating larger objects until one cannot fit into the hole */
910 for (addr = hole_start; addr < hole_end; ) {
911 struct i915_vma *vma;
912 u64 size = BIT_ULL(order++);
913
914 size = min(size, hole_end - addr);
915 obj = fake_dma_object(vm->i915, size);
916 if (IS_ERR(obj)) {
917 err = PTR_ERR(obj);
918 break;
919 }
920
921 list_add(&obj->st_link, &objects);
922
923 vma = i915_vma_instance(obj, vm, NULL);
924 if (IS_ERR(vma)) {
925 err = PTR_ERR(vma);
926 break;
927 }
928
929 GEM_BUG_ON(vma->size != size);
930
931 err = i915_vma_pin(vma, 0, 0, addr | flags);
932 if (err) {
933 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
934 __func__, addr, size, hole_start, hole_end, err);
935 break;
936 }
937
938 if (!drm_mm_node_allocated(&vma->node) ||
939 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
940 pr_err("%s incorrect at %llx + %llx\n",
941 __func__, addr, size);
942 i915_vma_unpin(vma);
943 err = i915_vma_unbind_unlocked(vma);
944 err = -EINVAL;
945 break;
946 }
947
948 i915_vma_unpin(vma);
949 addr += round_up(size, min_alignment);
950
951 /*
952 * Since we are injecting allocation faults at random intervals,
953 * wait for this allocation to complete before we change the
954 * faultinjection.
955 */
956 err = i915_vma_sync(vma);
957 if (err)
958 break;
959
960 if (igt_timeout(end_time,
961 "%s timed out at ofset %llx [%llx - %llx]\n",
962 __func__, addr, hole_start, hole_end)) {
963 err = -EINTR;
964 break;
965 }
966 }
967
968 close_object_list(&objects, vm);
969 cleanup_freed_objects(vm->i915);
970 return err;
971 }
972
shrink_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)973 static int shrink_hole(struct i915_address_space *vm,
974 u64 hole_start, u64 hole_end,
975 unsigned long end_time)
976 {
977 unsigned long prime;
978 int err;
979
980 vm->fault_attr.probability = 999;
981 atomic_set(&vm->fault_attr.times, -1);
982
983 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
984 vm->fault_attr.interval = prime;
985 err = __shrink_hole(vm, hole_start, hole_end, end_time);
986 if (err)
987 break;
988 }
989
990 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
991
992 return err;
993 }
994
shrink_boom(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)995 static int shrink_boom(struct i915_address_space *vm,
996 u64 hole_start, u64 hole_end,
997 unsigned long end_time)
998 {
999 unsigned int sizes[] = { SZ_2M, SZ_1G };
1000 struct drm_i915_gem_object *purge;
1001 struct drm_i915_gem_object *explode;
1002 int err;
1003 int i;
1004
1005 /*
1006 * Catch the case which shrink_hole seems to miss. The setup here
1007 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
1008 * ensuring that all vma assiocated with the respective pd/pdp are
1009 * unpinned at the time.
1010 */
1011
1012 for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1013 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1014 unsigned int size = sizes[i];
1015 struct i915_vma *vma;
1016
1017 purge = fake_dma_object(vm->i915, size);
1018 if (IS_ERR(purge))
1019 return PTR_ERR(purge);
1020
1021 vma = i915_vma_instance(purge, vm, NULL);
1022 if (IS_ERR(vma)) {
1023 err = PTR_ERR(vma);
1024 goto err_purge;
1025 }
1026
1027 err = i915_vma_pin(vma, 0, 0, flags);
1028 if (err)
1029 goto err_purge;
1030
1031 /* Should now be ripe for purging */
1032 i915_vma_unpin(vma);
1033
1034 explode = fake_dma_object(vm->i915, size);
1035 if (IS_ERR(explode)) {
1036 err = PTR_ERR(explode);
1037 goto err_purge;
1038 }
1039
1040 vm->fault_attr.probability = 100;
1041 vm->fault_attr.interval = 1;
1042 atomic_set(&vm->fault_attr.times, -1);
1043
1044 vma = i915_vma_instance(explode, vm, NULL);
1045 if (IS_ERR(vma)) {
1046 err = PTR_ERR(vma);
1047 goto err_explode;
1048 }
1049
1050 err = i915_vma_pin(vma, 0, 0, flags | size);
1051 if (err)
1052 goto err_explode;
1053
1054 i915_vma_unpin(vma);
1055
1056 i915_gem_object_put(purge);
1057 i915_gem_object_put(explode);
1058
1059 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1060 cleanup_freed_objects(vm->i915);
1061 }
1062
1063 return 0;
1064
1065 err_explode:
1066 i915_gem_object_put(explode);
1067 err_purge:
1068 i915_gem_object_put(purge);
1069 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1070 return err;
1071 }
1072
misaligned_case(struct i915_address_space * vm,struct intel_memory_region * mr,u64 addr,u64 size,unsigned long flags)1073 static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr,
1074 u64 addr, u64 size, unsigned long flags)
1075 {
1076 struct drm_i915_gem_object *obj;
1077 struct i915_vma *vma;
1078 int err = 0;
1079 u64 expected_vma_size, expected_node_size;
1080 bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM ||
1081 mr->type == INTEL_MEMORY_STOLEN_LOCAL;
1082
1083 obj = i915_gem_object_create_region(mr, size, 0, I915_BO_ALLOC_GPU_ONLY);
1084 if (IS_ERR(obj)) {
1085 /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */
1086 if (PTR_ERR(obj) == -ENODEV && is_stolen)
1087 return 0;
1088 return PTR_ERR(obj);
1089 }
1090
1091 vma = i915_vma_instance(obj, vm, NULL);
1092 if (IS_ERR(vma)) {
1093 err = PTR_ERR(vma);
1094 goto err_put;
1095 }
1096
1097 err = i915_vma_pin(vma, 0, 0, addr | flags);
1098 if (err)
1099 goto err_put;
1100 i915_vma_unpin(vma);
1101
1102 if (!drm_mm_node_allocated(&vma->node)) {
1103 err = -EINVAL;
1104 goto err_put;
1105 }
1106
1107 if (i915_vma_misplaced(vma, 0, 0, addr | flags)) {
1108 err = -EINVAL;
1109 goto err_put;
1110 }
1111
1112 expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1));
1113 expected_node_size = expected_vma_size;
1114
1115 if (HAS_64K_PAGES(vm->i915) && i915_gem_object_is_lmem(obj)) {
1116 /*
1117 * The compact-pt should expand lmem node to 2MB for the ppGTT,
1118 * for all other cases we should only expect 64K.
1119 */
1120 expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K);
1121 if (NEEDS_COMPACT_PT(vm->i915) && !i915_is_ggtt(vm))
1122 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1123 else
1124 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_64K);
1125 }
1126
1127 if (vma->size != expected_vma_size || vma->node.size != expected_node_size) {
1128 err = i915_vma_unbind_unlocked(vma);
1129 err = -EBADSLT;
1130 goto err_put;
1131 }
1132
1133 err = i915_vma_unbind_unlocked(vma);
1134 if (err)
1135 goto err_put;
1136
1137 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1138
1139 err_put:
1140 i915_gem_object_put(obj);
1141 cleanup_freed_objects(vm->i915);
1142 return err;
1143 }
1144
misaligned_pin(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)1145 static int misaligned_pin(struct i915_address_space *vm,
1146 u64 hole_start, u64 hole_end,
1147 unsigned long end_time)
1148 {
1149 struct intel_memory_region *mr;
1150 enum intel_region_id id;
1151 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
1152 int err = 0;
1153 u64 hole_size = hole_end - hole_start;
1154
1155 if (i915_is_ggtt(vm))
1156 flags |= PIN_GLOBAL;
1157
1158 for_each_memory_region(mr, vm->i915, id) {
1159 u64 min_alignment = i915_vm_min_alignment(vm, mr->type);
1160 u64 size = min_alignment;
1161 u64 addr = round_down(hole_start + (hole_size / 2), min_alignment);
1162
1163 /* avoid -ENOSPC on very small hole setups */
1164 if (hole_size < 3 * min_alignment)
1165 continue;
1166
1167 /* we can't test < 4k alignment due to flags being encoded in lower bits */
1168 if (min_alignment != I915_GTT_PAGE_SIZE_4K) {
1169 err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags);
1170 /* misaligned should error with -EINVAL*/
1171 if (!err)
1172 err = -EBADSLT;
1173 if (err != -EINVAL)
1174 return err;
1175 }
1176
1177 /* test for vma->size expansion to min page size */
1178 err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags);
1179 if (err)
1180 return err;
1181
1182 /* test for intermediate size not expanding vma->size for large alignments */
1183 err = misaligned_case(vm, mr, addr, size / 2, flags);
1184 if (err)
1185 return err;
1186 }
1187
1188 return 0;
1189 }
1190
exercise_ppgtt(struct drm_i915_private * dev_priv,int (* func)(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1191 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
1192 int (*func)(struct i915_address_space *vm,
1193 u64 hole_start, u64 hole_end,
1194 unsigned long end_time))
1195 {
1196 struct i915_ppgtt *ppgtt;
1197 IGT_TIMEOUT(end_time);
1198 struct file *file;
1199 int err;
1200
1201 if (!HAS_FULL_PPGTT(dev_priv))
1202 return 0;
1203
1204 file = mock_file(dev_priv);
1205 if (IS_ERR(file))
1206 return PTR_ERR(file);
1207
1208 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
1209 if (IS_ERR(ppgtt)) {
1210 err = PTR_ERR(ppgtt);
1211 goto out_free;
1212 }
1213 GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1214 assert_vm_alive(&ppgtt->vm);
1215
1216 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1217
1218 i915_vm_put(&ppgtt->vm);
1219
1220 out_free:
1221 fput(file);
1222 return err;
1223 }
1224
igt_ppgtt_fill(void * arg)1225 static int igt_ppgtt_fill(void *arg)
1226 {
1227 return exercise_ppgtt(arg, fill_hole);
1228 }
1229
igt_ppgtt_walk(void * arg)1230 static int igt_ppgtt_walk(void *arg)
1231 {
1232 return exercise_ppgtt(arg, walk_hole);
1233 }
1234
igt_ppgtt_pot(void * arg)1235 static int igt_ppgtt_pot(void *arg)
1236 {
1237 return exercise_ppgtt(arg, pot_hole);
1238 }
1239
igt_ppgtt_drunk(void * arg)1240 static int igt_ppgtt_drunk(void *arg)
1241 {
1242 return exercise_ppgtt(arg, drunk_hole);
1243 }
1244
igt_ppgtt_lowlevel(void * arg)1245 static int igt_ppgtt_lowlevel(void *arg)
1246 {
1247 return exercise_ppgtt(arg, lowlevel_hole);
1248 }
1249
igt_ppgtt_shrink(void * arg)1250 static int igt_ppgtt_shrink(void *arg)
1251 {
1252 return exercise_ppgtt(arg, shrink_hole);
1253 }
1254
igt_ppgtt_shrink_boom(void * arg)1255 static int igt_ppgtt_shrink_boom(void *arg)
1256 {
1257 return exercise_ppgtt(arg, shrink_boom);
1258 }
1259
igt_ppgtt_misaligned_pin(void * arg)1260 static int igt_ppgtt_misaligned_pin(void *arg)
1261 {
1262 return exercise_ppgtt(arg, misaligned_pin);
1263 }
1264
sort_holes(void * priv,const struct list_head * A,const struct list_head * B)1265 static int sort_holes(void *priv, const struct list_head *A,
1266 const struct list_head *B)
1267 {
1268 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1269 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1270
1271 if (a->start < b->start)
1272 return -1;
1273 else
1274 return 1;
1275 }
1276
exercise_ggtt(struct drm_i915_private * i915,int (* func)(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1277 static int exercise_ggtt(struct drm_i915_private *i915,
1278 int (*func)(struct i915_address_space *vm,
1279 u64 hole_start, u64 hole_end,
1280 unsigned long end_time))
1281 {
1282 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1283 u64 hole_start, hole_end, last = 0;
1284 struct drm_mm_node *node;
1285 IGT_TIMEOUT(end_time);
1286 int err = 0;
1287
1288 restart:
1289 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1290 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1291 if (hole_start < last)
1292 continue;
1293
1294 if (ggtt->vm.mm.color_adjust)
1295 ggtt->vm.mm.color_adjust(node, 0,
1296 &hole_start, &hole_end);
1297 if (hole_start >= hole_end)
1298 continue;
1299
1300 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1301 if (err)
1302 break;
1303
1304 /* As we have manipulated the drm_mm, the list may be corrupt */
1305 last = hole_end;
1306 goto restart;
1307 }
1308
1309 return err;
1310 }
1311
igt_ggtt_fill(void * arg)1312 static int igt_ggtt_fill(void *arg)
1313 {
1314 return exercise_ggtt(arg, fill_hole);
1315 }
1316
igt_ggtt_walk(void * arg)1317 static int igt_ggtt_walk(void *arg)
1318 {
1319 return exercise_ggtt(arg, walk_hole);
1320 }
1321
igt_ggtt_pot(void * arg)1322 static int igt_ggtt_pot(void *arg)
1323 {
1324 return exercise_ggtt(arg, pot_hole);
1325 }
1326
igt_ggtt_drunk(void * arg)1327 static int igt_ggtt_drunk(void *arg)
1328 {
1329 return exercise_ggtt(arg, drunk_hole);
1330 }
1331
igt_ggtt_lowlevel(void * arg)1332 static int igt_ggtt_lowlevel(void *arg)
1333 {
1334 return exercise_ggtt(arg, lowlevel_hole);
1335 }
1336
igt_ggtt_misaligned_pin(void * arg)1337 static int igt_ggtt_misaligned_pin(void *arg)
1338 {
1339 return exercise_ggtt(arg, misaligned_pin);
1340 }
1341
igt_ggtt_page(void * arg)1342 static int igt_ggtt_page(void *arg)
1343 {
1344 const unsigned int count = PAGE_SIZE/sizeof(u32);
1345 I915_RND_STATE(prng);
1346 struct drm_i915_private *i915 = arg;
1347 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1348 struct drm_i915_gem_object *obj;
1349 intel_wakeref_t wakeref;
1350 struct drm_mm_node tmp;
1351 unsigned int *order, n;
1352 int err;
1353
1354 if (!i915_ggtt_has_aperture(ggtt))
1355 return 0;
1356
1357 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1358 if (IS_ERR(obj))
1359 return PTR_ERR(obj);
1360
1361 err = i915_gem_object_pin_pages_unlocked(obj);
1362 if (err)
1363 goto out_free;
1364
1365 memset(&tmp, 0, sizeof(tmp));
1366 mutex_lock(&ggtt->vm.mutex);
1367 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1368 count * PAGE_SIZE, 0,
1369 I915_COLOR_UNEVICTABLE,
1370 0, ggtt->mappable_end,
1371 DRM_MM_INSERT_LOW);
1372 mutex_unlock(&ggtt->vm.mutex);
1373 if (err)
1374 goto out_unpin;
1375
1376 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1377
1378 for (n = 0; n < count; n++) {
1379 u64 offset = tmp.start + n * PAGE_SIZE;
1380
1381 ggtt->vm.insert_page(&ggtt->vm,
1382 i915_gem_object_get_dma_address(obj, 0),
1383 offset, I915_CACHE_NONE, 0);
1384 }
1385
1386 order = i915_random_order(count, &prng);
1387 if (!order) {
1388 err = -ENOMEM;
1389 goto out_remove;
1390 }
1391
1392 for (n = 0; n < count; n++) {
1393 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1394 u32 __iomem *vaddr;
1395
1396 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1397 iowrite32(n, vaddr + n);
1398 io_mapping_unmap_atomic(vaddr);
1399 }
1400 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1401
1402 i915_random_reorder(order, count, &prng);
1403 for (n = 0; n < count; n++) {
1404 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1405 u32 __iomem *vaddr;
1406 u32 val;
1407
1408 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1409 val = ioread32(vaddr + n);
1410 io_mapping_unmap_atomic(vaddr);
1411
1412 if (val != n) {
1413 pr_err("insert page failed: found %d, expected %d\n",
1414 val, n);
1415 err = -EINVAL;
1416 break;
1417 }
1418 }
1419
1420 kfree(order);
1421 out_remove:
1422 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1423 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1424 mutex_lock(&ggtt->vm.mutex);
1425 drm_mm_remove_node(&tmp);
1426 mutex_unlock(&ggtt->vm.mutex);
1427 out_unpin:
1428 i915_gem_object_unpin_pages(obj);
1429 out_free:
1430 i915_gem_object_put(obj);
1431 return err;
1432 }
1433
track_vma_bind(struct i915_vma * vma)1434 static void track_vma_bind(struct i915_vma *vma)
1435 {
1436 struct drm_i915_gem_object *obj = vma->obj;
1437
1438 __i915_gem_object_pin_pages(obj);
1439
1440 GEM_BUG_ON(atomic_read(&vma->pages_count));
1441 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1442 __i915_gem_object_pin_pages(obj);
1443 vma->pages = obj->mm.pages;
1444 vma->resource->bi.pages = vma->pages;
1445
1446 mutex_lock(&vma->vm->mutex);
1447 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1448 mutex_unlock(&vma->vm->mutex);
1449 }
1450
exercise_mock(struct drm_i915_private * i915,int (* func)(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1451 static int exercise_mock(struct drm_i915_private *i915,
1452 int (*func)(struct i915_address_space *vm,
1453 u64 hole_start, u64 hole_end,
1454 unsigned long end_time))
1455 {
1456 const u64 limit = totalram_pages() << PAGE_SHIFT;
1457 struct i915_address_space *vm;
1458 struct i915_gem_context *ctx;
1459 IGT_TIMEOUT(end_time);
1460 int err;
1461
1462 ctx = mock_context(i915, "mock");
1463 if (!ctx)
1464 return -ENOMEM;
1465
1466 vm = i915_gem_context_get_eb_vm(ctx);
1467 err = func(vm, 0, min(vm->total, limit), end_time);
1468 i915_vm_put(vm);
1469
1470 mock_context_close(ctx);
1471 return err;
1472 }
1473
igt_mock_fill(void * arg)1474 static int igt_mock_fill(void *arg)
1475 {
1476 struct i915_ggtt *ggtt = arg;
1477
1478 return exercise_mock(ggtt->vm.i915, fill_hole);
1479 }
1480
igt_mock_walk(void * arg)1481 static int igt_mock_walk(void *arg)
1482 {
1483 struct i915_ggtt *ggtt = arg;
1484
1485 return exercise_mock(ggtt->vm.i915, walk_hole);
1486 }
1487
igt_mock_pot(void * arg)1488 static int igt_mock_pot(void *arg)
1489 {
1490 struct i915_ggtt *ggtt = arg;
1491
1492 return exercise_mock(ggtt->vm.i915, pot_hole);
1493 }
1494
igt_mock_drunk(void * arg)1495 static int igt_mock_drunk(void *arg)
1496 {
1497 struct i915_ggtt *ggtt = arg;
1498
1499 return exercise_mock(ggtt->vm.i915, drunk_hole);
1500 }
1501
reserve_gtt_with_resource(struct i915_vma * vma,u64 offset)1502 static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset)
1503 {
1504 struct i915_address_space *vm = vma->vm;
1505 struct i915_vma_resource *vma_res;
1506 struct drm_i915_gem_object *obj = vma->obj;
1507 int err;
1508
1509 vma_res = i915_vma_resource_alloc();
1510 if (IS_ERR(vma_res))
1511 return PTR_ERR(vma_res);
1512
1513 mutex_lock(&vm->mutex);
1514 err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size,
1515 offset,
1516 obj->cache_level,
1517 0);
1518 if (!err) {
1519 i915_vma_resource_init_from_vma(vma_res, vma);
1520 vma->resource = vma_res;
1521 } else {
1522 kfree(vma_res);
1523 }
1524 mutex_unlock(&vm->mutex);
1525
1526 return err;
1527 }
1528
igt_gtt_reserve(void * arg)1529 static int igt_gtt_reserve(void *arg)
1530 {
1531 struct i915_ggtt *ggtt = arg;
1532 struct drm_i915_gem_object *obj, *on;
1533 I915_RND_STATE(prng);
1534 LIST_HEAD(objects);
1535 u64 total;
1536 int err = -ENODEV;
1537
1538 /* i915_gem_gtt_reserve() tries to reserve the precise range
1539 * for the node, and evicts if it has to. So our test checks that
1540 * it can give us the requsted space and prevent overlaps.
1541 */
1542
1543 /* Start by filling the GGTT */
1544 for (total = 0;
1545 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1546 total += 2 * I915_GTT_PAGE_SIZE) {
1547 struct i915_vma *vma;
1548
1549 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1550 2 * PAGE_SIZE);
1551 if (IS_ERR(obj)) {
1552 err = PTR_ERR(obj);
1553 goto out;
1554 }
1555
1556 err = i915_gem_object_pin_pages_unlocked(obj);
1557 if (err) {
1558 i915_gem_object_put(obj);
1559 goto out;
1560 }
1561
1562 list_add(&obj->st_link, &objects);
1563 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1564 if (IS_ERR(vma)) {
1565 err = PTR_ERR(vma);
1566 goto out;
1567 }
1568
1569 err = reserve_gtt_with_resource(vma, total);
1570 if (err) {
1571 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1572 total, ggtt->vm.total, err);
1573 goto out;
1574 }
1575 track_vma_bind(vma);
1576
1577 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1578 if (vma->node.start != total ||
1579 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1580 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1581 vma->node.start, vma->node.size,
1582 total, 2*I915_GTT_PAGE_SIZE);
1583 err = -EINVAL;
1584 goto out;
1585 }
1586 }
1587
1588 /* Now we start forcing evictions */
1589 for (total = I915_GTT_PAGE_SIZE;
1590 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1591 total += 2 * I915_GTT_PAGE_SIZE) {
1592 struct i915_vma *vma;
1593
1594 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1595 2 * PAGE_SIZE);
1596 if (IS_ERR(obj)) {
1597 err = PTR_ERR(obj);
1598 goto out;
1599 }
1600
1601 err = i915_gem_object_pin_pages_unlocked(obj);
1602 if (err) {
1603 i915_gem_object_put(obj);
1604 goto out;
1605 }
1606
1607 list_add(&obj->st_link, &objects);
1608
1609 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1610 if (IS_ERR(vma)) {
1611 err = PTR_ERR(vma);
1612 goto out;
1613 }
1614
1615 err = reserve_gtt_with_resource(vma, total);
1616 if (err) {
1617 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1618 total, ggtt->vm.total, err);
1619 goto out;
1620 }
1621 track_vma_bind(vma);
1622
1623 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1624 if (vma->node.start != total ||
1625 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1626 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1627 vma->node.start, vma->node.size,
1628 total, 2*I915_GTT_PAGE_SIZE);
1629 err = -EINVAL;
1630 goto out;
1631 }
1632 }
1633
1634 /* And then try at random */
1635 list_for_each_entry_safe(obj, on, &objects, st_link) {
1636 struct i915_vma *vma;
1637 u64 offset;
1638
1639 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1640 if (IS_ERR(vma)) {
1641 err = PTR_ERR(vma);
1642 goto out;
1643 }
1644
1645 err = i915_vma_unbind_unlocked(vma);
1646 if (err) {
1647 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1648 goto out;
1649 }
1650
1651 offset = igt_random_offset(&prng,
1652 0, ggtt->vm.total,
1653 2 * I915_GTT_PAGE_SIZE,
1654 I915_GTT_MIN_ALIGNMENT);
1655
1656 err = reserve_gtt_with_resource(vma, offset);
1657 if (err) {
1658 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1659 total, ggtt->vm.total, err);
1660 goto out;
1661 }
1662 track_vma_bind(vma);
1663
1664 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1665 if (vma->node.start != offset ||
1666 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1667 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1668 vma->node.start, vma->node.size,
1669 offset, 2*I915_GTT_PAGE_SIZE);
1670 err = -EINVAL;
1671 goto out;
1672 }
1673 }
1674
1675 out:
1676 list_for_each_entry_safe(obj, on, &objects, st_link) {
1677 i915_gem_object_unpin_pages(obj);
1678 i915_gem_object_put(obj);
1679 }
1680 return err;
1681 }
1682
insert_gtt_with_resource(struct i915_vma * vma)1683 static int insert_gtt_with_resource(struct i915_vma *vma)
1684 {
1685 struct i915_address_space *vm = vma->vm;
1686 struct i915_vma_resource *vma_res;
1687 struct drm_i915_gem_object *obj = vma->obj;
1688 int err;
1689
1690 vma_res = i915_vma_resource_alloc();
1691 if (IS_ERR(vma_res))
1692 return PTR_ERR(vma_res);
1693
1694 mutex_lock(&vm->mutex);
1695 err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0,
1696 obj->cache_level, 0, vm->total, 0);
1697 if (!err) {
1698 i915_vma_resource_init_from_vma(vma_res, vma);
1699 vma->resource = vma_res;
1700 } else {
1701 kfree(vma_res);
1702 }
1703 mutex_unlock(&vm->mutex);
1704
1705 return err;
1706 }
1707
igt_gtt_insert(void * arg)1708 static int igt_gtt_insert(void *arg)
1709 {
1710 struct i915_ggtt *ggtt = arg;
1711 struct drm_i915_gem_object *obj, *on;
1712 struct drm_mm_node tmp = {};
1713 const struct invalid_insert {
1714 u64 size;
1715 u64 alignment;
1716 u64 start, end;
1717 } invalid_insert[] = {
1718 {
1719 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1720 0, ggtt->vm.total,
1721 },
1722 {
1723 2*I915_GTT_PAGE_SIZE, 0,
1724 0, I915_GTT_PAGE_SIZE,
1725 },
1726 {
1727 -(u64)I915_GTT_PAGE_SIZE, 0,
1728 0, 4*I915_GTT_PAGE_SIZE,
1729 },
1730 {
1731 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1732 0, 4*I915_GTT_PAGE_SIZE,
1733 },
1734 {
1735 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1736 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1737 },
1738 {}
1739 }, *ii;
1740 LIST_HEAD(objects);
1741 u64 total;
1742 int err = -ENODEV;
1743
1744 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1745 * to the node, evicting if required.
1746 */
1747
1748 /* Check a couple of obviously invalid requests */
1749 for (ii = invalid_insert; ii->size; ii++) {
1750 mutex_lock(&ggtt->vm.mutex);
1751 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp,
1752 ii->size, ii->alignment,
1753 I915_COLOR_UNEVICTABLE,
1754 ii->start, ii->end,
1755 0);
1756 mutex_unlock(&ggtt->vm.mutex);
1757 if (err != -ENOSPC) {
1758 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1759 ii->size, ii->alignment, ii->start, ii->end,
1760 err);
1761 return -EINVAL;
1762 }
1763 }
1764
1765 /* Start by filling the GGTT */
1766 for (total = 0;
1767 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1768 total += I915_GTT_PAGE_SIZE) {
1769 struct i915_vma *vma;
1770
1771 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1772 I915_GTT_PAGE_SIZE);
1773 if (IS_ERR(obj)) {
1774 err = PTR_ERR(obj);
1775 goto out;
1776 }
1777
1778 err = i915_gem_object_pin_pages_unlocked(obj);
1779 if (err) {
1780 i915_gem_object_put(obj);
1781 goto out;
1782 }
1783
1784 list_add(&obj->st_link, &objects);
1785
1786 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1787 if (IS_ERR(vma)) {
1788 err = PTR_ERR(vma);
1789 goto out;
1790 }
1791
1792 err = insert_gtt_with_resource(vma);
1793 if (err == -ENOSPC) {
1794 /* maxed out the GGTT space */
1795 i915_gem_object_put(obj);
1796 break;
1797 }
1798 if (err) {
1799 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1800 total, ggtt->vm.total, err);
1801 goto out;
1802 }
1803 track_vma_bind(vma);
1804 __i915_vma_pin(vma);
1805
1806 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1807 }
1808
1809 list_for_each_entry(obj, &objects, st_link) {
1810 struct i915_vma *vma;
1811
1812 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1813 if (IS_ERR(vma)) {
1814 err = PTR_ERR(vma);
1815 goto out;
1816 }
1817
1818 if (!drm_mm_node_allocated(&vma->node)) {
1819 pr_err("VMA was unexpectedly evicted!\n");
1820 err = -EINVAL;
1821 goto out;
1822 }
1823
1824 __i915_vma_unpin(vma);
1825 }
1826
1827 /* If we then reinsert, we should find the same hole */
1828 list_for_each_entry_safe(obj, on, &objects, st_link) {
1829 struct i915_vma *vma;
1830 u64 offset;
1831
1832 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1833 if (IS_ERR(vma)) {
1834 err = PTR_ERR(vma);
1835 goto out;
1836 }
1837
1838 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1839 offset = vma->node.start;
1840
1841 err = i915_vma_unbind_unlocked(vma);
1842 if (err) {
1843 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1844 goto out;
1845 }
1846
1847 err = insert_gtt_with_resource(vma);
1848 if (err) {
1849 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1850 total, ggtt->vm.total, err);
1851 goto out;
1852 }
1853 track_vma_bind(vma);
1854
1855 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1856 if (vma->node.start != offset) {
1857 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1858 offset, vma->node.start);
1859 err = -EINVAL;
1860 goto out;
1861 }
1862 }
1863
1864 /* And then force evictions */
1865 for (total = 0;
1866 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1867 total += 2 * I915_GTT_PAGE_SIZE) {
1868 struct i915_vma *vma;
1869
1870 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1871 2 * I915_GTT_PAGE_SIZE);
1872 if (IS_ERR(obj)) {
1873 err = PTR_ERR(obj);
1874 goto out;
1875 }
1876
1877 err = i915_gem_object_pin_pages_unlocked(obj);
1878 if (err) {
1879 i915_gem_object_put(obj);
1880 goto out;
1881 }
1882
1883 list_add(&obj->st_link, &objects);
1884
1885 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1886 if (IS_ERR(vma)) {
1887 err = PTR_ERR(vma);
1888 goto out;
1889 }
1890
1891 err = insert_gtt_with_resource(vma);
1892 if (err) {
1893 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1894 total, ggtt->vm.total, err);
1895 goto out;
1896 }
1897 track_vma_bind(vma);
1898
1899 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1900 }
1901
1902 out:
1903 list_for_each_entry_safe(obj, on, &objects, st_link) {
1904 i915_gem_object_unpin_pages(obj);
1905 i915_gem_object_put(obj);
1906 }
1907 return err;
1908 }
1909
i915_gem_gtt_mock_selftests(void)1910 int i915_gem_gtt_mock_selftests(void)
1911 {
1912 static const struct i915_subtest tests[] = {
1913 SUBTEST(igt_mock_drunk),
1914 SUBTEST(igt_mock_walk),
1915 SUBTEST(igt_mock_pot),
1916 SUBTEST(igt_mock_fill),
1917 SUBTEST(igt_gtt_reserve),
1918 SUBTEST(igt_gtt_insert),
1919 };
1920 struct drm_i915_private *i915;
1921 struct intel_gt *gt;
1922 int err;
1923
1924 i915 = mock_gem_device();
1925 if (!i915)
1926 return -ENOMEM;
1927
1928 /* allocate the ggtt */
1929 err = intel_gt_assign_ggtt(to_gt(i915));
1930 if (err)
1931 goto out_put;
1932
1933 gt = to_gt(i915);
1934
1935 mock_init_ggtt(gt);
1936
1937 err = i915_subtests(tests, gt->ggtt);
1938
1939 mock_device_flush(i915);
1940 i915_gem_drain_freed_objects(i915);
1941 mock_fini_ggtt(gt->ggtt);
1942
1943 out_put:
1944 mock_destroy_device(i915);
1945 return err;
1946 }
1947
context_sync(struct intel_context * ce)1948 static int context_sync(struct intel_context *ce)
1949 {
1950 struct i915_request *rq;
1951 long timeout;
1952
1953 rq = intel_context_create_request(ce);
1954 if (IS_ERR(rq))
1955 return PTR_ERR(rq);
1956
1957 i915_request_get(rq);
1958 i915_request_add(rq);
1959
1960 timeout = i915_request_wait(rq, 0, HZ / 5);
1961 i915_request_put(rq);
1962
1963 return timeout < 0 ? -EIO : 0;
1964 }
1965
1966 static struct i915_request *
submit_batch(struct intel_context * ce,u64 addr)1967 submit_batch(struct intel_context *ce, u64 addr)
1968 {
1969 struct i915_request *rq;
1970 int err;
1971
1972 rq = intel_context_create_request(ce);
1973 if (IS_ERR(rq))
1974 return rq;
1975
1976 err = 0;
1977 if (rq->engine->emit_init_breadcrumb) /* detect a hang */
1978 err = rq->engine->emit_init_breadcrumb(rq);
1979 if (err == 0)
1980 err = rq->engine->emit_bb_start(rq, addr, 0, 0);
1981
1982 if (err == 0)
1983 i915_request_get(rq);
1984 i915_request_add(rq);
1985
1986 return err ? ERR_PTR(err) : rq;
1987 }
1988
spinner(u32 * batch,int i)1989 static u32 *spinner(u32 *batch, int i)
1990 {
1991 return batch + i * 64 / sizeof(*batch) + 4;
1992 }
1993
end_spin(u32 * batch,int i)1994 static void end_spin(u32 *batch, int i)
1995 {
1996 *spinner(batch, i) = MI_BATCH_BUFFER_END;
1997 wmb();
1998 }
1999
igt_cs_tlb(void * arg)2000 static int igt_cs_tlb(void *arg)
2001 {
2002 const unsigned int count = PAGE_SIZE / 64;
2003 const unsigned int chunk_size = count * PAGE_SIZE;
2004 struct drm_i915_private *i915 = arg;
2005 struct drm_i915_gem_object *bbe, *act, *out;
2006 struct i915_gem_engines_iter it;
2007 struct i915_address_space *vm;
2008 struct i915_gem_context *ctx;
2009 struct intel_context *ce;
2010 struct i915_vma *vma;
2011 I915_RND_STATE(prng);
2012 struct file *file;
2013 unsigned int i;
2014 u32 *result;
2015 u32 *batch;
2016 int err = 0;
2017
2018 /*
2019 * Our mission here is to fool the hardware to execute something
2020 * from scratch as it has not seen the batch move (due to missing
2021 * the TLB invalidate).
2022 */
2023
2024 file = mock_file(i915);
2025 if (IS_ERR(file))
2026 return PTR_ERR(file);
2027
2028 ctx = live_context(i915, file);
2029 if (IS_ERR(ctx)) {
2030 err = PTR_ERR(ctx);
2031 goto out_unlock;
2032 }
2033
2034 vm = i915_gem_context_get_eb_vm(ctx);
2035 if (i915_is_ggtt(vm))
2036 goto out_vm;
2037
2038 /* Create two pages; dummy we prefill the TLB, and intended */
2039 bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
2040 if (IS_ERR(bbe)) {
2041 err = PTR_ERR(bbe);
2042 goto out_vm;
2043 }
2044
2045 batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC);
2046 if (IS_ERR(batch)) {
2047 err = PTR_ERR(batch);
2048 goto out_put_bbe;
2049 }
2050 memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
2051 i915_gem_object_flush_map(bbe);
2052 i915_gem_object_unpin_map(bbe);
2053
2054 act = i915_gem_object_create_internal(i915, PAGE_SIZE);
2055 if (IS_ERR(act)) {
2056 err = PTR_ERR(act);
2057 goto out_put_bbe;
2058 }
2059
2060 /* Track the execution of each request by writing into different slot */
2061 batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC);
2062 if (IS_ERR(batch)) {
2063 err = PTR_ERR(batch);
2064 goto out_put_act;
2065 }
2066 for (i = 0; i < count; i++) {
2067 u32 *cs = batch + i * 64 / sizeof(*cs);
2068 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
2069
2070 GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
2071 cs[0] = MI_STORE_DWORD_IMM_GEN4;
2072 if (GRAPHICS_VER(i915) >= 8) {
2073 cs[1] = lower_32_bits(addr);
2074 cs[2] = upper_32_bits(addr);
2075 cs[3] = i;
2076 cs[4] = MI_NOOP;
2077 cs[5] = MI_BATCH_BUFFER_START_GEN8;
2078 } else {
2079 cs[1] = 0;
2080 cs[2] = lower_32_bits(addr);
2081 cs[3] = i;
2082 cs[4] = MI_NOOP;
2083 cs[5] = MI_BATCH_BUFFER_START;
2084 }
2085 }
2086
2087 out = i915_gem_object_create_internal(i915, PAGE_SIZE);
2088 if (IS_ERR(out)) {
2089 err = PTR_ERR(out);
2090 goto out_put_batch;
2091 }
2092 i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
2093
2094 vma = i915_vma_instance(out, vm, NULL);
2095 if (IS_ERR(vma)) {
2096 err = PTR_ERR(vma);
2097 goto out_put_out;
2098 }
2099
2100 err = i915_vma_pin(vma, 0, 0,
2101 PIN_USER |
2102 PIN_OFFSET_FIXED |
2103 (vm->total - PAGE_SIZE));
2104 if (err)
2105 goto out_put_out;
2106 GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
2107
2108 result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB);
2109 if (IS_ERR(result)) {
2110 err = PTR_ERR(result);
2111 goto out_put_out;
2112 }
2113
2114 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
2115 IGT_TIMEOUT(end_time);
2116 unsigned long pass = 0;
2117
2118 if (!intel_engine_can_store_dword(ce->engine))
2119 continue;
2120
2121 while (!__igt_timeout(end_time, NULL)) {
2122 struct i915_vm_pt_stash stash = {};
2123 struct i915_request *rq;
2124 struct i915_gem_ww_ctx ww;
2125 struct i915_vma_resource *vma_res;
2126 u64 offset;
2127
2128 offset = igt_random_offset(&prng,
2129 0, vm->total - PAGE_SIZE,
2130 chunk_size, PAGE_SIZE);
2131
2132 memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
2133
2134 vma = i915_vma_instance(bbe, vm, NULL);
2135 if (IS_ERR(vma)) {
2136 err = PTR_ERR(vma);
2137 goto end;
2138 }
2139
2140 i915_gem_object_lock(bbe, NULL);
2141 err = i915_vma_get_pages(vma);
2142 i915_gem_object_unlock(bbe);
2143 if (err)
2144 goto end;
2145
2146 vma_res = i915_vma_resource_alloc();
2147 if (IS_ERR(vma_res)) {
2148 i915_vma_put_pages(vma);
2149 err = PTR_ERR(vma_res);
2150 goto end;
2151 }
2152
2153 i915_gem_ww_ctx_init(&ww, false);
2154 retry:
2155 err = i915_vm_lock_objects(vm, &ww);
2156 if (err)
2157 goto end_ww;
2158
2159 err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
2160 if (err)
2161 goto end_ww;
2162
2163 err = i915_vm_map_pt_stash(vm, &stash);
2164 if (!err)
2165 vm->allocate_va_range(vm, &stash, offset, chunk_size);
2166 i915_vm_free_pt_stash(vm, &stash);
2167 end_ww:
2168 if (err == -EDEADLK) {
2169 err = i915_gem_ww_ctx_backoff(&ww);
2170 if (!err)
2171 goto retry;
2172 }
2173 i915_gem_ww_ctx_fini(&ww);
2174 if (err) {
2175 kfree(vma_res);
2176 goto end;
2177 }
2178
2179 i915_vma_resource_init_from_vma(vma_res, vma);
2180 /* Prime the TLB with the dummy pages */
2181 for (i = 0; i < count; i++) {
2182 vma_res->start = offset + i * PAGE_SIZE;
2183 vm->insert_entries(vm, vma_res, I915_CACHE_NONE,
2184 0);
2185
2186 rq = submit_batch(ce, vma_res->start);
2187 if (IS_ERR(rq)) {
2188 err = PTR_ERR(rq);
2189 i915_vma_resource_fini(vma_res);
2190 kfree(vma_res);
2191 goto end;
2192 }
2193 i915_request_put(rq);
2194 }
2195 i915_vma_resource_fini(vma_res);
2196 i915_vma_put_pages(vma);
2197
2198 err = context_sync(ce);
2199 if (err) {
2200 pr_err("%s: dummy setup timed out\n",
2201 ce->engine->name);
2202 kfree(vma_res);
2203 goto end;
2204 }
2205
2206 vma = i915_vma_instance(act, vm, NULL);
2207 if (IS_ERR(vma)) {
2208 kfree(vma_res);
2209 err = PTR_ERR(vma);
2210 goto end;
2211 }
2212
2213 i915_gem_object_lock(act, NULL);
2214 err = i915_vma_get_pages(vma);
2215 i915_gem_object_unlock(act);
2216 if (err) {
2217 kfree(vma_res);
2218 goto end;
2219 }
2220
2221 i915_vma_resource_init_from_vma(vma_res, vma);
2222 /* Replace the TLB with target batches */
2223 for (i = 0; i < count; i++) {
2224 struct i915_request *rq;
2225 u32 *cs = batch + i * 64 / sizeof(*cs);
2226 u64 addr;
2227
2228 vma_res->start = offset + i * PAGE_SIZE;
2229 vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 0);
2230
2231 addr = vma_res->start + i * 64;
2232 cs[4] = MI_NOOP;
2233 cs[6] = lower_32_bits(addr);
2234 cs[7] = upper_32_bits(addr);
2235 wmb();
2236
2237 rq = submit_batch(ce, addr);
2238 if (IS_ERR(rq)) {
2239 err = PTR_ERR(rq);
2240 i915_vma_resource_fini(vma_res);
2241 kfree(vma_res);
2242 goto end;
2243 }
2244
2245 /* Wait until the context chain has started */
2246 if (i == 0) {
2247 while (READ_ONCE(result[i]) &&
2248 !i915_request_completed(rq))
2249 cond_resched();
2250 } else {
2251 end_spin(batch, i - 1);
2252 }
2253
2254 i915_request_put(rq);
2255 }
2256 end_spin(batch, count - 1);
2257
2258 i915_vma_resource_fini(vma_res);
2259 kfree(vma_res);
2260 i915_vma_put_pages(vma);
2261
2262 err = context_sync(ce);
2263 if (err) {
2264 pr_err("%s: writes timed out\n",
2265 ce->engine->name);
2266 goto end;
2267 }
2268
2269 for (i = 0; i < count; i++) {
2270 if (result[i] != i) {
2271 pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
2272 ce->engine->name, pass,
2273 offset, i, result[i], i);
2274 err = -EINVAL;
2275 goto end;
2276 }
2277 }
2278
2279 vm->clear_range(vm, offset, chunk_size);
2280 pass++;
2281 }
2282 }
2283 end:
2284 if (igt_flush_test(i915))
2285 err = -EIO;
2286 i915_gem_context_unlock_engines(ctx);
2287 i915_gem_object_unpin_map(out);
2288 out_put_out:
2289 i915_gem_object_put(out);
2290 out_put_batch:
2291 i915_gem_object_unpin_map(act);
2292 out_put_act:
2293 i915_gem_object_put(act);
2294 out_put_bbe:
2295 i915_gem_object_put(bbe);
2296 out_vm:
2297 i915_vm_put(vm);
2298 out_unlock:
2299 fput(file);
2300 return err;
2301 }
2302
i915_gem_gtt_live_selftests(struct drm_i915_private * i915)2303 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
2304 {
2305 static const struct i915_subtest tests[] = {
2306 SUBTEST(igt_ppgtt_alloc),
2307 SUBTEST(igt_ppgtt_lowlevel),
2308 SUBTEST(igt_ppgtt_drunk),
2309 SUBTEST(igt_ppgtt_walk),
2310 SUBTEST(igt_ppgtt_pot),
2311 SUBTEST(igt_ppgtt_fill),
2312 SUBTEST(igt_ppgtt_shrink),
2313 SUBTEST(igt_ppgtt_shrink_boom),
2314 SUBTEST(igt_ppgtt_misaligned_pin),
2315 SUBTEST(igt_ggtt_lowlevel),
2316 SUBTEST(igt_ggtt_drunk),
2317 SUBTEST(igt_ggtt_walk),
2318 SUBTEST(igt_ggtt_pot),
2319 SUBTEST(igt_ggtt_fill),
2320 SUBTEST(igt_ggtt_page),
2321 SUBTEST(igt_ggtt_misaligned_pin),
2322 SUBTEST(igt_cs_tlb),
2323 };
2324
2325 GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total));
2326
2327 return i915_live_subtests(tests, i915);
2328 }
2329
