1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <linux/list_sort.h>
26 #include <linux/prime_numbers.h>
27
28 #include "../i915_selftest.h"
29 #include "i915_random.h"
30
31 #include "mock_context.h"
32 #include "mock_drm.h"
33 #include "mock_gem_device.h"
34
cleanup_freed_objects(struct drm_i915_private * i915)35 static void cleanup_freed_objects(struct drm_i915_private *i915)
36 {
37 /*
38 * As we may hold onto the struct_mutex for inordinate lengths of
39 * time, the NMI khungtaskd detector may fire for the free objects
40 * worker.
41 */
42 mutex_unlock(&i915->drm.struct_mutex);
43
44 i915_gem_drain_freed_objects(i915);
45
46 mutex_lock(&i915->drm.struct_mutex);
47 }
48
fake_free_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)49 static void fake_free_pages(struct drm_i915_gem_object *obj,
50 struct sg_table *pages)
51 {
52 sg_free_table(pages);
53 kfree(pages);
54 }
55
fake_get_pages(struct drm_i915_gem_object * obj)56 static int fake_get_pages(struct drm_i915_gem_object *obj)
57 {
58 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
59 #define PFN_BIAS 0x1000
60 struct sg_table *pages;
61 struct scatterlist *sg;
62 unsigned int sg_page_sizes;
63 typeof(obj->base.size) rem;
64
65 pages = kmalloc(sizeof(*pages), GFP);
66 if (!pages)
67 return -ENOMEM;
68
69 rem = round_up(obj->base.size, BIT(31)) >> 31;
70 if (sg_alloc_table(pages, rem, GFP)) {
71 kfree(pages);
72 return -ENOMEM;
73 }
74
75 sg_page_sizes = 0;
76 rem = obj->base.size;
77 for (sg = pages->sgl; sg; sg = sg_next(sg)) {
78 unsigned long len = min_t(typeof(rem), rem, BIT(31));
79
80 GEM_BUG_ON(!len);
81 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
82 sg_dma_address(sg) = page_to_phys(sg_page(sg));
83 sg_dma_len(sg) = len;
84 sg_page_sizes |= len;
85
86 rem -= len;
87 }
88 GEM_BUG_ON(rem);
89
90 obj->mm.madv = I915_MADV_DONTNEED;
91
92 __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
93
94 return 0;
95 #undef GFP
96 }
97
fake_put_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)98 static void fake_put_pages(struct drm_i915_gem_object *obj,
99 struct sg_table *pages)
100 {
101 fake_free_pages(obj, pages);
102 obj->mm.dirty = false;
103 obj->mm.madv = I915_MADV_WILLNEED;
104 }
105
106 static const struct drm_i915_gem_object_ops fake_ops = {
107 .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
108 .get_pages = fake_get_pages,
109 .put_pages = fake_put_pages,
110 };
111
112 static struct drm_i915_gem_object *
fake_dma_object(struct drm_i915_private * i915,u64 size)113 fake_dma_object(struct drm_i915_private *i915, u64 size)
114 {
115 struct drm_i915_gem_object *obj;
116
117 GEM_BUG_ON(!size);
118 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
119
120 if (overflows_type(size, obj->base.size))
121 return ERR_PTR(-E2BIG);
122
123 obj = i915_gem_object_alloc(i915);
124 if (!obj)
125 goto err;
126
127 drm_gem_private_object_init(&i915->drm, &obj->base, size);
128 i915_gem_object_init(obj, &fake_ops);
129
130 obj->write_domain = I915_GEM_DOMAIN_CPU;
131 obj->read_domains = I915_GEM_DOMAIN_CPU;
132 obj->cache_level = I915_CACHE_NONE;
133
134 /* Preallocate the "backing storage" */
135 if (i915_gem_object_pin_pages(obj))
136 goto err_obj;
137
138 i915_gem_object_unpin_pages(obj);
139 return obj;
140
141 err_obj:
142 i915_gem_object_put(obj);
143 err:
144 return ERR_PTR(-ENOMEM);
145 }
146
igt_ppgtt_alloc(void * arg)147 static int igt_ppgtt_alloc(void *arg)
148 {
149 struct drm_i915_private *dev_priv = arg;
150 struct i915_hw_ppgtt *ppgtt;
151 u64 size, last, limit;
152 int err = 0;
153
154 /* Allocate a ppggt and try to fill the entire range */
155
156 if (!USES_PPGTT(dev_priv))
157 return 0;
158
159 ppgtt = __hw_ppgtt_create(dev_priv);
160 if (IS_ERR(ppgtt))
161 return PTR_ERR(ppgtt);
162
163 if (!ppgtt->vm.allocate_va_range)
164 goto err_ppgtt_cleanup;
165
166 /*
167 * While we only allocate the page tables here and so we could
168 * address a much larger GTT than we could actually fit into
169 * RAM, a practical limit is the amount of physical pages in the system.
170 * This should ensure that we do not run into the oomkiller during
171 * the test and take down the machine wilfully.
172 */
173 limit = totalram_pages << PAGE_SHIFT;
174 limit = min(ppgtt->vm.total, limit);
175
176 /* Check we can allocate the entire range */
177 for (size = 4096; size <= limit; size <<= 2) {
178 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size);
179 if (err) {
180 if (err == -ENOMEM) {
181 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n",
182 size, ilog2(size));
183 err = 0; /* virtual space too large! */
184 }
185 goto err_ppgtt_cleanup;
186 }
187
188 cond_resched();
189
190 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
191 }
192
193 /* Check we can incrementally allocate the entire range */
194 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
195 err = ppgtt->vm.allocate_va_range(&ppgtt->vm,
196 last, size - last);
197 if (err) {
198 if (err == -ENOMEM) {
199 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n",
200 last, size - last, ilog2(size));
201 err = 0; /* virtual space too large! */
202 }
203 goto err_ppgtt_cleanup;
204 }
205
206 cond_resched();
207 }
208
209 err_ppgtt_cleanup:
210 mutex_lock(&dev_priv->drm.struct_mutex);
211 i915_ppgtt_put(ppgtt);
212 mutex_unlock(&dev_priv->drm.struct_mutex);
213 return err;
214 }
215
lowlevel_hole(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)216 static int lowlevel_hole(struct drm_i915_private *i915,
217 struct i915_address_space *vm,
218 u64 hole_start, u64 hole_end,
219 unsigned long end_time)
220 {
221 I915_RND_STATE(seed_prng);
222 unsigned int size;
223 struct i915_vma mock_vma;
224
225 memset(&mock_vma, 0, sizeof(struct i915_vma));
226
227 /* Keep creating larger objects until one cannot fit into the hole */
228 for (size = 12; (hole_end - hole_start) >> size; size++) {
229 I915_RND_SUBSTATE(prng, seed_prng);
230 struct drm_i915_gem_object *obj;
231 unsigned int *order, count, n;
232 u64 hole_size;
233
234 hole_size = (hole_end - hole_start) >> size;
235 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
236 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
237 count = hole_size >> 1;
238 if (!count) {
239 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
240 __func__, hole_start, hole_end, size, hole_size);
241 break;
242 }
243
244 do {
245 order = i915_random_order(count, &prng);
246 if (order)
247 break;
248 } while (count >>= 1);
249 if (!count)
250 return -ENOMEM;
251 GEM_BUG_ON(!order);
252
253 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
254 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
255
256 /* Ignore allocation failures (i.e. don't report them as
257 * a test failure) as we are purposefully allocating very
258 * large objects without checking that we have sufficient
259 * memory. We expect to hit -ENOMEM.
260 */
261
262 obj = fake_dma_object(i915, BIT_ULL(size));
263 if (IS_ERR(obj)) {
264 kfree(order);
265 break;
266 }
267
268 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
269
270 if (i915_gem_object_pin_pages(obj)) {
271 i915_gem_object_put(obj);
272 kfree(order);
273 break;
274 }
275
276 for (n = 0; n < count; n++) {
277 u64 addr = hole_start + order[n] * BIT_ULL(size);
278
279 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
280
281 if (igt_timeout(end_time,
282 "%s timed out before %d/%d\n",
283 __func__, n, count)) {
284 hole_end = hole_start; /* quit */
285 break;
286 }
287
288 if (vm->allocate_va_range &&
289 vm->allocate_va_range(vm, addr, BIT_ULL(size)))
290 break;
291
292 mock_vma.pages = obj->mm.pages;
293 mock_vma.node.size = BIT_ULL(size);
294 mock_vma.node.start = addr;
295
296 intel_runtime_pm_get(i915);
297 vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0);
298 intel_runtime_pm_put(i915);
299 }
300 count = n;
301
302 i915_random_reorder(order, count, &prng);
303 for (n = 0; n < count; n++) {
304 u64 addr = hole_start + order[n] * BIT_ULL(size);
305
306 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
307 vm->clear_range(vm, addr, BIT_ULL(size));
308 }
309
310 i915_gem_object_unpin_pages(obj);
311 i915_gem_object_put(obj);
312
313 kfree(order);
314
315 cleanup_freed_objects(i915);
316 }
317
318 return 0;
319 }
320
close_object_list(struct list_head * objects,struct i915_address_space * vm)321 static void close_object_list(struct list_head *objects,
322 struct i915_address_space *vm)
323 {
324 struct drm_i915_gem_object *obj, *on;
325 int ignored;
326
327 list_for_each_entry_safe(obj, on, objects, st_link) {
328 struct i915_vma *vma;
329
330 vma = i915_vma_instance(obj, vm, NULL);
331 if (!IS_ERR(vma))
332 ignored = i915_vma_unbind(vma);
333 /* Only ppgtt vma may be closed before the object is freed */
334 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma))
335 i915_vma_close(vma);
336
337 list_del(&obj->st_link);
338 i915_gem_object_put(obj);
339 }
340 }
341
fill_hole(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)342 static int fill_hole(struct drm_i915_private *i915,
343 struct i915_address_space *vm,
344 u64 hole_start, u64 hole_end,
345 unsigned long end_time)
346 {
347 const u64 hole_size = hole_end - hole_start;
348 struct drm_i915_gem_object *obj;
349 const unsigned long max_pages =
350 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
351 const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
352 unsigned long npages, prime, flags;
353 struct i915_vma *vma;
354 LIST_HEAD(objects);
355 int err;
356
357 /* Try binding many VMA working inwards from either edge */
358
359 flags = PIN_OFFSET_FIXED | PIN_USER;
360 if (i915_is_ggtt(vm))
361 flags |= PIN_GLOBAL;
362
363 for_each_prime_number_from(prime, 2, max_step) {
364 for (npages = 1; npages <= max_pages; npages *= prime) {
365 const u64 full_size = npages << PAGE_SHIFT;
366 const struct {
367 const char *name;
368 u64 offset;
369 int step;
370 } phases[] = {
371 { "top-down", hole_end, -1, },
372 { "bottom-up", hole_start, 1, },
373 { }
374 }, *p;
375
376 obj = fake_dma_object(i915, full_size);
377 if (IS_ERR(obj))
378 break;
379
380 list_add(&obj->st_link, &objects);
381
382 /* Align differing sized objects against the edges, and
383 * check we don't walk off into the void when binding
384 * them into the GTT.
385 */
386 for (p = phases; p->name; p++) {
387 u64 offset;
388
389 offset = p->offset;
390 list_for_each_entry(obj, &objects, st_link) {
391 vma = i915_vma_instance(obj, vm, NULL);
392 if (IS_ERR(vma))
393 continue;
394
395 if (p->step < 0) {
396 if (offset < hole_start + obj->base.size)
397 break;
398 offset -= obj->base.size;
399 }
400
401 err = i915_vma_pin(vma, 0, 0, offset | flags);
402 if (err) {
403 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
404 __func__, p->name, err, npages, prime, offset);
405 goto err;
406 }
407
408 if (!drm_mm_node_allocated(&vma->node) ||
409 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
410 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
411 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
412 offset);
413 err = -EINVAL;
414 goto err;
415 }
416
417 i915_vma_unpin(vma);
418
419 if (p->step > 0) {
420 if (offset + obj->base.size > hole_end)
421 break;
422 offset += obj->base.size;
423 }
424 }
425
426 offset = p->offset;
427 list_for_each_entry(obj, &objects, st_link) {
428 vma = i915_vma_instance(obj, vm, NULL);
429 if (IS_ERR(vma))
430 continue;
431
432 if (p->step < 0) {
433 if (offset < hole_start + obj->base.size)
434 break;
435 offset -= obj->base.size;
436 }
437
438 if (!drm_mm_node_allocated(&vma->node) ||
439 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
440 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
441 __func__, p->name, vma->node.start, vma->node.size,
442 offset);
443 err = -EINVAL;
444 goto err;
445 }
446
447 err = i915_vma_unbind(vma);
448 if (err) {
449 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
450 __func__, p->name, vma->node.start, vma->node.size,
451 err);
452 goto err;
453 }
454
455 if (p->step > 0) {
456 if (offset + obj->base.size > hole_end)
457 break;
458 offset += obj->base.size;
459 }
460 }
461
462 offset = p->offset;
463 list_for_each_entry_reverse(obj, &objects, st_link) {
464 vma = i915_vma_instance(obj, vm, NULL);
465 if (IS_ERR(vma))
466 continue;
467
468 if (p->step < 0) {
469 if (offset < hole_start + obj->base.size)
470 break;
471 offset -= obj->base.size;
472 }
473
474 err = i915_vma_pin(vma, 0, 0, offset | flags);
475 if (err) {
476 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
477 __func__, p->name, err, npages, prime, offset);
478 goto err;
479 }
480
481 if (!drm_mm_node_allocated(&vma->node) ||
482 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
483 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
484 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
485 offset);
486 err = -EINVAL;
487 goto err;
488 }
489
490 i915_vma_unpin(vma);
491
492 if (p->step > 0) {
493 if (offset + obj->base.size > hole_end)
494 break;
495 offset += obj->base.size;
496 }
497 }
498
499 offset = p->offset;
500 list_for_each_entry_reverse(obj, &objects, st_link) {
501 vma = i915_vma_instance(obj, vm, NULL);
502 if (IS_ERR(vma))
503 continue;
504
505 if (p->step < 0) {
506 if (offset < hole_start + obj->base.size)
507 break;
508 offset -= obj->base.size;
509 }
510
511 if (!drm_mm_node_allocated(&vma->node) ||
512 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
513 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
514 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
515 offset);
516 err = -EINVAL;
517 goto err;
518 }
519
520 err = i915_vma_unbind(vma);
521 if (err) {
522 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
523 __func__, p->name, vma->node.start, vma->node.size,
524 err);
525 goto err;
526 }
527
528 if (p->step > 0) {
529 if (offset + obj->base.size > hole_end)
530 break;
531 offset += obj->base.size;
532 }
533 }
534 }
535
536 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
537 __func__, npages, prime)) {
538 err = -EINTR;
539 goto err;
540 }
541 }
542
543 close_object_list(&objects, vm);
544 cleanup_freed_objects(i915);
545 }
546
547 return 0;
548
549 err:
550 close_object_list(&objects, vm);
551 return err;
552 }
553
walk_hole(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)554 static int walk_hole(struct drm_i915_private *i915,
555 struct i915_address_space *vm,
556 u64 hole_start, u64 hole_end,
557 unsigned long end_time)
558 {
559 const u64 hole_size = hole_end - hole_start;
560 const unsigned long max_pages =
561 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
562 unsigned long flags;
563 u64 size;
564
565 /* Try binding a single VMA in different positions within the hole */
566
567 flags = PIN_OFFSET_FIXED | PIN_USER;
568 if (i915_is_ggtt(vm))
569 flags |= PIN_GLOBAL;
570
571 for_each_prime_number_from(size, 1, max_pages) {
572 struct drm_i915_gem_object *obj;
573 struct i915_vma *vma;
574 u64 addr;
575 int err = 0;
576
577 obj = fake_dma_object(i915, size << PAGE_SHIFT);
578 if (IS_ERR(obj))
579 break;
580
581 vma = i915_vma_instance(obj, vm, NULL);
582 if (IS_ERR(vma)) {
583 err = PTR_ERR(vma);
584 goto err_put;
585 }
586
587 for (addr = hole_start;
588 addr + obj->base.size < hole_end;
589 addr += obj->base.size) {
590 err = i915_vma_pin(vma, 0, 0, addr | flags);
591 if (err) {
592 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
593 __func__, addr, vma->size,
594 hole_start, hole_end, err);
595 goto err_close;
596 }
597 i915_vma_unpin(vma);
598
599 if (!drm_mm_node_allocated(&vma->node) ||
600 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
601 pr_err("%s incorrect at %llx + %llx\n",
602 __func__, addr, vma->size);
603 err = -EINVAL;
604 goto err_close;
605 }
606
607 err = i915_vma_unbind(vma);
608 if (err) {
609 pr_err("%s unbind failed at %llx + %llx with err=%d\n",
610 __func__, addr, vma->size, err);
611 goto err_close;
612 }
613
614 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
615
616 if (igt_timeout(end_time,
617 "%s timed out at %llx\n",
618 __func__, addr)) {
619 err = -EINTR;
620 goto err_close;
621 }
622 }
623
624 err_close:
625 if (!i915_vma_is_ggtt(vma))
626 i915_vma_close(vma);
627 err_put:
628 i915_gem_object_put(obj);
629 if (err)
630 return err;
631
632 cleanup_freed_objects(i915);
633 }
634
635 return 0;
636 }
637
pot_hole(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)638 static int pot_hole(struct drm_i915_private *i915,
639 struct i915_address_space *vm,
640 u64 hole_start, u64 hole_end,
641 unsigned long end_time)
642 {
643 struct drm_i915_gem_object *obj;
644 struct i915_vma *vma;
645 unsigned long flags;
646 unsigned int pot;
647 int err = 0;
648
649 flags = PIN_OFFSET_FIXED | PIN_USER;
650 if (i915_is_ggtt(vm))
651 flags |= PIN_GLOBAL;
652
653 obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE);
654 if (IS_ERR(obj))
655 return PTR_ERR(obj);
656
657 vma = i915_vma_instance(obj, vm, NULL);
658 if (IS_ERR(vma)) {
659 err = PTR_ERR(vma);
660 goto err_obj;
661 }
662
663 /* Insert a pair of pages across every pot boundary within the hole */
664 for (pot = fls64(hole_end - 1) - 1;
665 pot > ilog2(2 * I915_GTT_PAGE_SIZE);
666 pot--) {
667 u64 step = BIT_ULL(pot);
668 u64 addr;
669
670 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
671 addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
672 addr += step) {
673 err = i915_vma_pin(vma, 0, 0, addr | flags);
674 if (err) {
675 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
676 __func__,
677 addr,
678 hole_start, hole_end,
679 err);
680 goto err;
681 }
682
683 if (!drm_mm_node_allocated(&vma->node) ||
684 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
685 pr_err("%s incorrect at %llx + %llx\n",
686 __func__, addr, vma->size);
687 i915_vma_unpin(vma);
688 err = i915_vma_unbind(vma);
689 err = -EINVAL;
690 goto err;
691 }
692
693 i915_vma_unpin(vma);
694 err = i915_vma_unbind(vma);
695 GEM_BUG_ON(err);
696 }
697
698 if (igt_timeout(end_time,
699 "%s timed out after %d/%d\n",
700 __func__, pot, fls64(hole_end - 1) - 1)) {
701 err = -EINTR;
702 goto err;
703 }
704 }
705
706 err:
707 if (!i915_vma_is_ggtt(vma))
708 i915_vma_close(vma);
709 err_obj:
710 i915_gem_object_put(obj);
711 return err;
712 }
713
drunk_hole(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)714 static int drunk_hole(struct drm_i915_private *i915,
715 struct i915_address_space *vm,
716 u64 hole_start, u64 hole_end,
717 unsigned long end_time)
718 {
719 I915_RND_STATE(prng);
720 unsigned int size;
721 unsigned long flags;
722
723 flags = PIN_OFFSET_FIXED | PIN_USER;
724 if (i915_is_ggtt(vm))
725 flags |= PIN_GLOBAL;
726
727 /* Keep creating larger objects until one cannot fit into the hole */
728 for (size = 12; (hole_end - hole_start) >> size; size++) {
729 struct drm_i915_gem_object *obj;
730 unsigned int *order, count, n;
731 struct i915_vma *vma;
732 u64 hole_size;
733 int err = -ENODEV;
734
735 hole_size = (hole_end - hole_start) >> size;
736 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
737 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
738 count = hole_size >> 1;
739 if (!count) {
740 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
741 __func__, hole_start, hole_end, size, hole_size);
742 break;
743 }
744
745 do {
746 order = i915_random_order(count, &prng);
747 if (order)
748 break;
749 } while (count >>= 1);
750 if (!count)
751 return -ENOMEM;
752 GEM_BUG_ON(!order);
753
754 /* Ignore allocation failures (i.e. don't report them as
755 * a test failure) as we are purposefully allocating very
756 * large objects without checking that we have sufficient
757 * memory. We expect to hit -ENOMEM.
758 */
759
760 obj = fake_dma_object(i915, BIT_ULL(size));
761 if (IS_ERR(obj)) {
762 kfree(order);
763 break;
764 }
765
766 vma = i915_vma_instance(obj, vm, NULL);
767 if (IS_ERR(vma)) {
768 err = PTR_ERR(vma);
769 goto err_obj;
770 }
771
772 GEM_BUG_ON(vma->size != BIT_ULL(size));
773
774 for (n = 0; n < count; n++) {
775 u64 addr = hole_start + order[n] * BIT_ULL(size);
776
777 err = i915_vma_pin(vma, 0, 0, addr | flags);
778 if (err) {
779 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
780 __func__,
781 addr, BIT_ULL(size),
782 hole_start, hole_end,
783 err);
784 goto err;
785 }
786
787 if (!drm_mm_node_allocated(&vma->node) ||
788 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
789 pr_err("%s incorrect at %llx + %llx\n",
790 __func__, addr, BIT_ULL(size));
791 i915_vma_unpin(vma);
792 err = i915_vma_unbind(vma);
793 err = -EINVAL;
794 goto err;
795 }
796
797 i915_vma_unpin(vma);
798 err = i915_vma_unbind(vma);
799 GEM_BUG_ON(err);
800
801 if (igt_timeout(end_time,
802 "%s timed out after %d/%d\n",
803 __func__, n, count)) {
804 err = -EINTR;
805 goto err;
806 }
807 }
808
809 err:
810 if (!i915_vma_is_ggtt(vma))
811 i915_vma_close(vma);
812 err_obj:
813 i915_gem_object_put(obj);
814 kfree(order);
815 if (err)
816 return err;
817
818 cleanup_freed_objects(i915);
819 }
820
821 return 0;
822 }
823
__shrink_hole(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)824 static int __shrink_hole(struct drm_i915_private *i915,
825 struct i915_address_space *vm,
826 u64 hole_start, u64 hole_end,
827 unsigned long end_time)
828 {
829 struct drm_i915_gem_object *obj;
830 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
831 unsigned int order = 12;
832 LIST_HEAD(objects);
833 int err = 0;
834 u64 addr;
835
836 /* Keep creating larger objects until one cannot fit into the hole */
837 for (addr = hole_start; addr < hole_end; ) {
838 struct i915_vma *vma;
839 u64 size = BIT_ULL(order++);
840
841 size = min(size, hole_end - addr);
842 obj = fake_dma_object(i915, size);
843 if (IS_ERR(obj)) {
844 err = PTR_ERR(obj);
845 break;
846 }
847
848 list_add(&obj->st_link, &objects);
849
850 vma = i915_vma_instance(obj, vm, NULL);
851 if (IS_ERR(vma)) {
852 err = PTR_ERR(vma);
853 break;
854 }
855
856 GEM_BUG_ON(vma->size != size);
857
858 err = i915_vma_pin(vma, 0, 0, addr | flags);
859 if (err) {
860 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
861 __func__, addr, size, hole_start, hole_end, err);
862 break;
863 }
864
865 if (!drm_mm_node_allocated(&vma->node) ||
866 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
867 pr_err("%s incorrect at %llx + %llx\n",
868 __func__, addr, size);
869 i915_vma_unpin(vma);
870 err = i915_vma_unbind(vma);
871 err = -EINVAL;
872 break;
873 }
874
875 i915_vma_unpin(vma);
876 addr += size;
877
878 if (igt_timeout(end_time,
879 "%s timed out at ofset %llx [%llx - %llx]\n",
880 __func__, addr, hole_start, hole_end)) {
881 err = -EINTR;
882 break;
883 }
884 }
885
886 close_object_list(&objects, vm);
887 cleanup_freed_objects(i915);
888 return err;
889 }
890
shrink_hole(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)891 static int shrink_hole(struct drm_i915_private *i915,
892 struct i915_address_space *vm,
893 u64 hole_start, u64 hole_end,
894 unsigned long end_time)
895 {
896 unsigned long prime;
897 int err;
898
899 vm->fault_attr.probability = 999;
900 atomic_set(&vm->fault_attr.times, -1);
901
902 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
903 vm->fault_attr.interval = prime;
904 err = __shrink_hole(i915, vm, hole_start, hole_end, end_time);
905 if (err)
906 break;
907 }
908
909 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
910
911 return err;
912 }
913
shrink_boom(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)914 static int shrink_boom(struct drm_i915_private *i915,
915 struct i915_address_space *vm,
916 u64 hole_start, u64 hole_end,
917 unsigned long end_time)
918 {
919 unsigned int sizes[] = { SZ_2M, SZ_1G };
920 struct drm_i915_gem_object *purge;
921 struct drm_i915_gem_object *explode;
922 int err;
923 int i;
924
925 /*
926 * Catch the case which shrink_hole seems to miss. The setup here
927 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
928 * ensuring that all vma assiocated with the respective pd/pdp are
929 * unpinned at the time.
930 */
931
932 for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
933 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
934 unsigned int size = sizes[i];
935 struct i915_vma *vma;
936
937 purge = fake_dma_object(i915, size);
938 if (IS_ERR(purge))
939 return PTR_ERR(purge);
940
941 vma = i915_vma_instance(purge, vm, NULL);
942 if (IS_ERR(vma)) {
943 err = PTR_ERR(vma);
944 goto err_purge;
945 }
946
947 err = i915_vma_pin(vma, 0, 0, flags);
948 if (err)
949 goto err_purge;
950
951 /* Should now be ripe for purging */
952 i915_vma_unpin(vma);
953
954 explode = fake_dma_object(i915, size);
955 if (IS_ERR(explode)) {
956 err = PTR_ERR(explode);
957 goto err_purge;
958 }
959
960 vm->fault_attr.probability = 100;
961 vm->fault_attr.interval = 1;
962 atomic_set(&vm->fault_attr.times, -1);
963
964 vma = i915_vma_instance(explode, vm, NULL);
965 if (IS_ERR(vma)) {
966 err = PTR_ERR(vma);
967 goto err_explode;
968 }
969
970 err = i915_vma_pin(vma, 0, 0, flags | size);
971 if (err)
972 goto err_explode;
973
974 i915_vma_unpin(vma);
975
976 i915_gem_object_put(purge);
977 i915_gem_object_put(explode);
978
979 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
980 cleanup_freed_objects(i915);
981 }
982
983 return 0;
984
985 err_explode:
986 i915_gem_object_put(explode);
987 err_purge:
988 i915_gem_object_put(purge);
989 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
990 return err;
991 }
992
exercise_ppgtt(struct drm_i915_private * dev_priv,int (* func)(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))993 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
994 int (*func)(struct drm_i915_private *i915,
995 struct i915_address_space *vm,
996 u64 hole_start, u64 hole_end,
997 unsigned long end_time))
998 {
999 struct drm_file *file;
1000 struct i915_hw_ppgtt *ppgtt;
1001 IGT_TIMEOUT(end_time);
1002 int err;
1003
1004 if (!USES_FULL_PPGTT(dev_priv))
1005 return 0;
1006
1007 file = mock_file(dev_priv);
1008 if (IS_ERR(file))
1009 return PTR_ERR(file);
1010
1011 mutex_lock(&dev_priv->drm.struct_mutex);
1012 ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv);
1013 if (IS_ERR(ppgtt)) {
1014 err = PTR_ERR(ppgtt);
1015 goto out_unlock;
1016 }
1017 GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1018 GEM_BUG_ON(ppgtt->vm.closed);
1019
1020 err = func(dev_priv, &ppgtt->vm, 0, ppgtt->vm.total, end_time);
1021
1022 i915_ppgtt_close(&ppgtt->vm);
1023 i915_ppgtt_put(ppgtt);
1024 out_unlock:
1025 mutex_unlock(&dev_priv->drm.struct_mutex);
1026
1027 mock_file_free(dev_priv, file);
1028 return err;
1029 }
1030
igt_ppgtt_fill(void * arg)1031 static int igt_ppgtt_fill(void *arg)
1032 {
1033 return exercise_ppgtt(arg, fill_hole);
1034 }
1035
igt_ppgtt_walk(void * arg)1036 static int igt_ppgtt_walk(void *arg)
1037 {
1038 return exercise_ppgtt(arg, walk_hole);
1039 }
1040
igt_ppgtt_pot(void * arg)1041 static int igt_ppgtt_pot(void *arg)
1042 {
1043 return exercise_ppgtt(arg, pot_hole);
1044 }
1045
igt_ppgtt_drunk(void * arg)1046 static int igt_ppgtt_drunk(void *arg)
1047 {
1048 return exercise_ppgtt(arg, drunk_hole);
1049 }
1050
igt_ppgtt_lowlevel(void * arg)1051 static int igt_ppgtt_lowlevel(void *arg)
1052 {
1053 return exercise_ppgtt(arg, lowlevel_hole);
1054 }
1055
igt_ppgtt_shrink(void * arg)1056 static int igt_ppgtt_shrink(void *arg)
1057 {
1058 return exercise_ppgtt(arg, shrink_hole);
1059 }
1060
igt_ppgtt_shrink_boom(void * arg)1061 static int igt_ppgtt_shrink_boom(void *arg)
1062 {
1063 return exercise_ppgtt(arg, shrink_boom);
1064 }
1065
sort_holes(void * priv,struct list_head * A,struct list_head * B)1066 static int sort_holes(void *priv, struct list_head *A, struct list_head *B)
1067 {
1068 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1069 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1070
1071 if (a->start < b->start)
1072 return -1;
1073 else
1074 return 1;
1075 }
1076
exercise_ggtt(struct drm_i915_private * i915,int (* func)(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1077 static int exercise_ggtt(struct drm_i915_private *i915,
1078 int (*func)(struct drm_i915_private *i915,
1079 struct i915_address_space *vm,
1080 u64 hole_start, u64 hole_end,
1081 unsigned long end_time))
1082 {
1083 struct i915_ggtt *ggtt = &i915->ggtt;
1084 u64 hole_start, hole_end, last = 0;
1085 struct drm_mm_node *node;
1086 IGT_TIMEOUT(end_time);
1087 int err = 0;
1088
1089 mutex_lock(&i915->drm.struct_mutex);
1090 restart:
1091 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1092 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1093 if (hole_start < last)
1094 continue;
1095
1096 if (ggtt->vm.mm.color_adjust)
1097 ggtt->vm.mm.color_adjust(node, 0,
1098 &hole_start, &hole_end);
1099 if (hole_start >= hole_end)
1100 continue;
1101
1102 err = func(i915, &ggtt->vm, hole_start, hole_end, end_time);
1103 if (err)
1104 break;
1105
1106 /* As we have manipulated the drm_mm, the list may be corrupt */
1107 last = hole_end;
1108 goto restart;
1109 }
1110 mutex_unlock(&i915->drm.struct_mutex);
1111
1112 return err;
1113 }
1114
igt_ggtt_fill(void * arg)1115 static int igt_ggtt_fill(void *arg)
1116 {
1117 return exercise_ggtt(arg, fill_hole);
1118 }
1119
igt_ggtt_walk(void * arg)1120 static int igt_ggtt_walk(void *arg)
1121 {
1122 return exercise_ggtt(arg, walk_hole);
1123 }
1124
igt_ggtt_pot(void * arg)1125 static int igt_ggtt_pot(void *arg)
1126 {
1127 return exercise_ggtt(arg, pot_hole);
1128 }
1129
igt_ggtt_drunk(void * arg)1130 static int igt_ggtt_drunk(void *arg)
1131 {
1132 return exercise_ggtt(arg, drunk_hole);
1133 }
1134
igt_ggtt_lowlevel(void * arg)1135 static int igt_ggtt_lowlevel(void *arg)
1136 {
1137 return exercise_ggtt(arg, lowlevel_hole);
1138 }
1139
igt_ggtt_page(void * arg)1140 static int igt_ggtt_page(void *arg)
1141 {
1142 const unsigned int count = PAGE_SIZE/sizeof(u32);
1143 I915_RND_STATE(prng);
1144 struct drm_i915_private *i915 = arg;
1145 struct i915_ggtt *ggtt = &i915->ggtt;
1146 struct drm_i915_gem_object *obj;
1147 struct drm_mm_node tmp;
1148 unsigned int *order, n;
1149 int err;
1150
1151 mutex_lock(&i915->drm.struct_mutex);
1152
1153 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1154 if (IS_ERR(obj)) {
1155 err = PTR_ERR(obj);
1156 goto out_unlock;
1157 }
1158
1159 err = i915_gem_object_pin_pages(obj);
1160 if (err)
1161 goto out_free;
1162
1163 memset(&tmp, 0, sizeof(tmp));
1164 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1165 count * PAGE_SIZE, 0,
1166 I915_COLOR_UNEVICTABLE,
1167 0, ggtt->mappable_end,
1168 DRM_MM_INSERT_LOW);
1169 if (err)
1170 goto out_unpin;
1171
1172 intel_runtime_pm_get(i915);
1173
1174 for (n = 0; n < count; n++) {
1175 u64 offset = tmp.start + n * PAGE_SIZE;
1176
1177 ggtt->vm.insert_page(&ggtt->vm,
1178 i915_gem_object_get_dma_address(obj, 0),
1179 offset, I915_CACHE_NONE, 0);
1180 }
1181
1182 order = i915_random_order(count, &prng);
1183 if (!order) {
1184 err = -ENOMEM;
1185 goto out_remove;
1186 }
1187
1188 for (n = 0; n < count; n++) {
1189 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1190 u32 __iomem *vaddr;
1191
1192 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1193 iowrite32(n, vaddr + n);
1194 io_mapping_unmap_atomic(vaddr);
1195 }
1196 i915_gem_flush_ggtt_writes(i915);
1197
1198 i915_random_reorder(order, count, &prng);
1199 for (n = 0; n < count; n++) {
1200 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1201 u32 __iomem *vaddr;
1202 u32 val;
1203
1204 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1205 val = ioread32(vaddr + n);
1206 io_mapping_unmap_atomic(vaddr);
1207
1208 if (val != n) {
1209 pr_err("insert page failed: found %d, expected %d\n",
1210 val, n);
1211 err = -EINVAL;
1212 break;
1213 }
1214 }
1215
1216 kfree(order);
1217 out_remove:
1218 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1219 intel_runtime_pm_put(i915);
1220 drm_mm_remove_node(&tmp);
1221 out_unpin:
1222 i915_gem_object_unpin_pages(obj);
1223 out_free:
1224 i915_gem_object_put(obj);
1225 out_unlock:
1226 mutex_unlock(&i915->drm.struct_mutex);
1227 return err;
1228 }
1229
track_vma_bind(struct i915_vma * vma)1230 static void track_vma_bind(struct i915_vma *vma)
1231 {
1232 struct drm_i915_gem_object *obj = vma->obj;
1233
1234 obj->bind_count++; /* track for eviction later */
1235 __i915_gem_object_pin_pages(obj);
1236
1237 vma->pages = obj->mm.pages;
1238 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1239 }
1240
exercise_mock(struct drm_i915_private * i915,int (* func)(struct drm_i915_private * i915,struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1241 static int exercise_mock(struct drm_i915_private *i915,
1242 int (*func)(struct drm_i915_private *i915,
1243 struct i915_address_space *vm,
1244 u64 hole_start, u64 hole_end,
1245 unsigned long end_time))
1246 {
1247 const u64 limit = totalram_pages << PAGE_SHIFT;
1248 struct i915_gem_context *ctx;
1249 struct i915_hw_ppgtt *ppgtt;
1250 IGT_TIMEOUT(end_time);
1251 int err;
1252
1253 ctx = mock_context(i915, "mock");
1254 if (!ctx)
1255 return -ENOMEM;
1256
1257 ppgtt = ctx->ppgtt;
1258 GEM_BUG_ON(!ppgtt);
1259
1260 err = func(i915, &ppgtt->vm, 0, min(ppgtt->vm.total, limit), end_time);
1261
1262 mock_context_close(ctx);
1263 return err;
1264 }
1265
igt_mock_fill(void * arg)1266 static int igt_mock_fill(void *arg)
1267 {
1268 return exercise_mock(arg, fill_hole);
1269 }
1270
igt_mock_walk(void * arg)1271 static int igt_mock_walk(void *arg)
1272 {
1273 return exercise_mock(arg, walk_hole);
1274 }
1275
igt_mock_pot(void * arg)1276 static int igt_mock_pot(void *arg)
1277 {
1278 return exercise_mock(arg, pot_hole);
1279 }
1280
igt_mock_drunk(void * arg)1281 static int igt_mock_drunk(void *arg)
1282 {
1283 return exercise_mock(arg, drunk_hole);
1284 }
1285
igt_gtt_reserve(void * arg)1286 static int igt_gtt_reserve(void *arg)
1287 {
1288 struct drm_i915_private *i915 = arg;
1289 struct drm_i915_gem_object *obj, *on;
1290 LIST_HEAD(objects);
1291 u64 total;
1292 int err = -ENODEV;
1293
1294 /* i915_gem_gtt_reserve() tries to reserve the precise range
1295 * for the node, and evicts if it has to. So our test checks that
1296 * it can give us the requsted space and prevent overlaps.
1297 */
1298
1299 /* Start by filling the GGTT */
1300 for (total = 0;
1301 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
1302 total += 2*I915_GTT_PAGE_SIZE) {
1303 struct i915_vma *vma;
1304
1305 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
1306 if (IS_ERR(obj)) {
1307 err = PTR_ERR(obj);
1308 goto out;
1309 }
1310
1311 err = i915_gem_object_pin_pages(obj);
1312 if (err) {
1313 i915_gem_object_put(obj);
1314 goto out;
1315 }
1316
1317 list_add(&obj->st_link, &objects);
1318
1319 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1320 if (IS_ERR(vma)) {
1321 err = PTR_ERR(vma);
1322 goto out;
1323 }
1324
1325 err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node,
1326 obj->base.size,
1327 total,
1328 obj->cache_level,
1329 0);
1330 if (err) {
1331 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1332 total, i915->ggtt.vm.total, err);
1333 goto out;
1334 }
1335 track_vma_bind(vma);
1336
1337 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1338 if (vma->node.start != total ||
1339 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1340 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1341 vma->node.start, vma->node.size,
1342 total, 2*I915_GTT_PAGE_SIZE);
1343 err = -EINVAL;
1344 goto out;
1345 }
1346 }
1347
1348 /* Now we start forcing evictions */
1349 for (total = I915_GTT_PAGE_SIZE;
1350 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
1351 total += 2*I915_GTT_PAGE_SIZE) {
1352 struct i915_vma *vma;
1353
1354 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
1355 if (IS_ERR(obj)) {
1356 err = PTR_ERR(obj);
1357 goto out;
1358 }
1359
1360 err = i915_gem_object_pin_pages(obj);
1361 if (err) {
1362 i915_gem_object_put(obj);
1363 goto out;
1364 }
1365
1366 list_add(&obj->st_link, &objects);
1367
1368 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1369 if (IS_ERR(vma)) {
1370 err = PTR_ERR(vma);
1371 goto out;
1372 }
1373
1374 err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node,
1375 obj->base.size,
1376 total,
1377 obj->cache_level,
1378 0);
1379 if (err) {
1380 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1381 total, i915->ggtt.vm.total, err);
1382 goto out;
1383 }
1384 track_vma_bind(vma);
1385
1386 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1387 if (vma->node.start != total ||
1388 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1389 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1390 vma->node.start, vma->node.size,
1391 total, 2*I915_GTT_PAGE_SIZE);
1392 err = -EINVAL;
1393 goto out;
1394 }
1395 }
1396
1397 /* And then try at random */
1398 list_for_each_entry_safe(obj, on, &objects, st_link) {
1399 struct i915_vma *vma;
1400 u64 offset;
1401
1402 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1403 if (IS_ERR(vma)) {
1404 err = PTR_ERR(vma);
1405 goto out;
1406 }
1407
1408 err = i915_vma_unbind(vma);
1409 if (err) {
1410 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1411 goto out;
1412 }
1413
1414 offset = random_offset(0, i915->ggtt.vm.total,
1415 2*I915_GTT_PAGE_SIZE,
1416 I915_GTT_MIN_ALIGNMENT);
1417
1418 err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node,
1419 obj->base.size,
1420 offset,
1421 obj->cache_level,
1422 0);
1423 if (err) {
1424 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1425 total, i915->ggtt.vm.total, err);
1426 goto out;
1427 }
1428 track_vma_bind(vma);
1429
1430 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1431 if (vma->node.start != offset ||
1432 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1433 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %lx)\n",
1434 vma->node.start, vma->node.size,
1435 offset, 2*I915_GTT_PAGE_SIZE);
1436 err = -EINVAL;
1437 goto out;
1438 }
1439 }
1440
1441 out:
1442 list_for_each_entry_safe(obj, on, &objects, st_link) {
1443 i915_gem_object_unpin_pages(obj);
1444 i915_gem_object_put(obj);
1445 }
1446 return err;
1447 }
1448
igt_gtt_insert(void * arg)1449 static int igt_gtt_insert(void *arg)
1450 {
1451 struct drm_i915_private *i915 = arg;
1452 struct drm_i915_gem_object *obj, *on;
1453 struct drm_mm_node tmp = {};
1454 const struct invalid_insert {
1455 u64 size;
1456 u64 alignment;
1457 u64 start, end;
1458 } invalid_insert[] = {
1459 {
1460 i915->ggtt.vm.total + I915_GTT_PAGE_SIZE, 0,
1461 0, i915->ggtt.vm.total,
1462 },
1463 {
1464 2*I915_GTT_PAGE_SIZE, 0,
1465 0, I915_GTT_PAGE_SIZE,
1466 },
1467 {
1468 -(u64)I915_GTT_PAGE_SIZE, 0,
1469 0, 4*I915_GTT_PAGE_SIZE,
1470 },
1471 {
1472 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1473 0, 4*I915_GTT_PAGE_SIZE,
1474 },
1475 {
1476 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1477 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1478 },
1479 {}
1480 }, *ii;
1481 LIST_HEAD(objects);
1482 u64 total;
1483 int err = -ENODEV;
1484
1485 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1486 * to the node, evicting if required.
1487 */
1488
1489 /* Check a couple of obviously invalid requests */
1490 for (ii = invalid_insert; ii->size; ii++) {
1491 err = i915_gem_gtt_insert(&i915->ggtt.vm, &tmp,
1492 ii->size, ii->alignment,
1493 I915_COLOR_UNEVICTABLE,
1494 ii->start, ii->end,
1495 0);
1496 if (err != -ENOSPC) {
1497 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1498 ii->size, ii->alignment, ii->start, ii->end,
1499 err);
1500 return -EINVAL;
1501 }
1502 }
1503
1504 /* Start by filling the GGTT */
1505 for (total = 0;
1506 total + I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
1507 total += I915_GTT_PAGE_SIZE) {
1508 struct i915_vma *vma;
1509
1510 obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
1511 if (IS_ERR(obj)) {
1512 err = PTR_ERR(obj);
1513 goto out;
1514 }
1515
1516 err = i915_gem_object_pin_pages(obj);
1517 if (err) {
1518 i915_gem_object_put(obj);
1519 goto out;
1520 }
1521
1522 list_add(&obj->st_link, &objects);
1523
1524 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1525 if (IS_ERR(vma)) {
1526 err = PTR_ERR(vma);
1527 goto out;
1528 }
1529
1530 err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node,
1531 obj->base.size, 0, obj->cache_level,
1532 0, i915->ggtt.vm.total,
1533 0);
1534 if (err == -ENOSPC) {
1535 /* maxed out the GGTT space */
1536 i915_gem_object_put(obj);
1537 break;
1538 }
1539 if (err) {
1540 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1541 total, i915->ggtt.vm.total, err);
1542 goto out;
1543 }
1544 track_vma_bind(vma);
1545 __i915_vma_pin(vma);
1546
1547 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1548 }
1549
1550 list_for_each_entry(obj, &objects, st_link) {
1551 struct i915_vma *vma;
1552
1553 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1554 if (IS_ERR(vma)) {
1555 err = PTR_ERR(vma);
1556 goto out;
1557 }
1558
1559 if (!drm_mm_node_allocated(&vma->node)) {
1560 pr_err("VMA was unexpectedly evicted!\n");
1561 err = -EINVAL;
1562 goto out;
1563 }
1564
1565 __i915_vma_unpin(vma);
1566 }
1567
1568 /* If we then reinsert, we should find the same hole */
1569 list_for_each_entry_safe(obj, on, &objects, st_link) {
1570 struct i915_vma *vma;
1571 u64 offset;
1572
1573 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1574 if (IS_ERR(vma)) {
1575 err = PTR_ERR(vma);
1576 goto out;
1577 }
1578
1579 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1580 offset = vma->node.start;
1581
1582 err = i915_vma_unbind(vma);
1583 if (err) {
1584 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1585 goto out;
1586 }
1587
1588 err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node,
1589 obj->base.size, 0, obj->cache_level,
1590 0, i915->ggtt.vm.total,
1591 0);
1592 if (err) {
1593 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1594 total, i915->ggtt.vm.total, err);
1595 goto out;
1596 }
1597 track_vma_bind(vma);
1598
1599 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1600 if (vma->node.start != offset) {
1601 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1602 offset, vma->node.start);
1603 err = -EINVAL;
1604 goto out;
1605 }
1606 }
1607
1608 /* And then force evictions */
1609 for (total = 0;
1610 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
1611 total += 2*I915_GTT_PAGE_SIZE) {
1612 struct i915_vma *vma;
1613
1614 obj = i915_gem_object_create_internal(i915, 2*I915_GTT_PAGE_SIZE);
1615 if (IS_ERR(obj)) {
1616 err = PTR_ERR(obj);
1617 goto out;
1618 }
1619
1620 err = i915_gem_object_pin_pages(obj);
1621 if (err) {
1622 i915_gem_object_put(obj);
1623 goto out;
1624 }
1625
1626 list_add(&obj->st_link, &objects);
1627
1628 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1629 if (IS_ERR(vma)) {
1630 err = PTR_ERR(vma);
1631 goto out;
1632 }
1633
1634 err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node,
1635 obj->base.size, 0, obj->cache_level,
1636 0, i915->ggtt.vm.total,
1637 0);
1638 if (err) {
1639 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1640 total, i915->ggtt.vm.total, err);
1641 goto out;
1642 }
1643 track_vma_bind(vma);
1644
1645 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1646 }
1647
1648 out:
1649 list_for_each_entry_safe(obj, on, &objects, st_link) {
1650 i915_gem_object_unpin_pages(obj);
1651 i915_gem_object_put(obj);
1652 }
1653 return err;
1654 }
1655
i915_gem_gtt_mock_selftests(void)1656 int i915_gem_gtt_mock_selftests(void)
1657 {
1658 static const struct i915_subtest tests[] = {
1659 SUBTEST(igt_mock_drunk),
1660 SUBTEST(igt_mock_walk),
1661 SUBTEST(igt_mock_pot),
1662 SUBTEST(igt_mock_fill),
1663 SUBTEST(igt_gtt_reserve),
1664 SUBTEST(igt_gtt_insert),
1665 };
1666 struct drm_i915_private *i915;
1667 int err;
1668
1669 i915 = mock_gem_device();
1670 if (!i915)
1671 return -ENOMEM;
1672
1673 mutex_lock(&i915->drm.struct_mutex);
1674 err = i915_subtests(tests, i915);
1675 mutex_unlock(&i915->drm.struct_mutex);
1676
1677 drm_dev_put(&i915->drm);
1678 return err;
1679 }
1680
i915_gem_gtt_live_selftests(struct drm_i915_private * i915)1681 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
1682 {
1683 static const struct i915_subtest tests[] = {
1684 SUBTEST(igt_ppgtt_alloc),
1685 SUBTEST(igt_ppgtt_lowlevel),
1686 SUBTEST(igt_ppgtt_drunk),
1687 SUBTEST(igt_ppgtt_walk),
1688 SUBTEST(igt_ppgtt_pot),
1689 SUBTEST(igt_ppgtt_fill),
1690 SUBTEST(igt_ppgtt_shrink),
1691 SUBTEST(igt_ppgtt_shrink_boom),
1692 SUBTEST(igt_ggtt_lowlevel),
1693 SUBTEST(igt_ggtt_drunk),
1694 SUBTEST(igt_ggtt_walk),
1695 SUBTEST(igt_ggtt_pot),
1696 SUBTEST(igt_ggtt_fill),
1697 SUBTEST(igt_ggtt_page),
1698 };
1699
1700 GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total));
1701
1702 return i915_subtests(tests, i915);
1703 }
1704
