1 /*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <drm/drmP.h>
35 #include <drm/amdgpu_drm.h>
36 #include <drm/drm_cache.h>
37 #include "amdgpu.h"
38 #include "amdgpu_trace.h"
39 #include "amdgpu_amdkfd.h"
40
41 /**
42 * DOC: amdgpu_object
43 *
44 * This defines the interfaces to operate on an &amdgpu_bo buffer object which
45 * represents memory used by driver (VRAM, system memory, etc.). The driver
46 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
47 * to create/destroy/set buffer object which are then managed by the kernel TTM
48 * memory manager.
49 * The interfaces are also used internally by kernel clients, including gfx,
50 * uvd, etc. for kernel managed allocations used by the GPU.
51 *
52 */
53
amdgpu_bo_need_backup(struct amdgpu_device * adev)54 static bool amdgpu_bo_need_backup(struct amdgpu_device *adev)
55 {
56 if (adev->flags & AMD_IS_APU)
57 return false;
58
59 if (amdgpu_gpu_recovery == 0 ||
60 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev)))
61 return false;
62
63 return true;
64 }
65
66 /**
67 * amdgpu_bo_subtract_pin_size - Remove BO from pin_size accounting
68 *
69 * @bo: &amdgpu_bo buffer object
70 *
71 * This function is called when a BO stops being pinned, and updates the
72 * &amdgpu_device pin_size values accordingly.
73 */
amdgpu_bo_subtract_pin_size(struct amdgpu_bo * bo)74 static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo)
75 {
76 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
77
78 if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
79 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
80 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
81 &adev->visible_pin_size);
82 } else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
83 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
84 }
85 }
86
amdgpu_bo_destroy(struct ttm_buffer_object * tbo)87 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
88 {
89 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
90 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
91
92 if (bo->pin_count > 0)
93 amdgpu_bo_subtract_pin_size(bo);
94
95 if (bo->kfd_bo)
96 amdgpu_amdkfd_unreserve_system_memory_limit(bo);
97
98 amdgpu_bo_kunmap(bo);
99
100 if (bo->gem_base.import_attach)
101 drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
102 drm_gem_object_release(&bo->gem_base);
103 amdgpu_bo_unref(&bo->parent);
104 if (!list_empty(&bo->shadow_list)) {
105 mutex_lock(&adev->shadow_list_lock);
106 list_del_init(&bo->shadow_list);
107 mutex_unlock(&adev->shadow_list_lock);
108 }
109 kfree(bo->metadata);
110 kfree(bo);
111 }
112
113 /**
114 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
115 * @bo: buffer object to be checked
116 *
117 * Uses destroy function associated with the object to determine if this is
118 * an &amdgpu_bo.
119 *
120 * Returns:
121 * true if the object belongs to &amdgpu_bo, false if not.
122 */
amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object * bo)123 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
124 {
125 if (bo->destroy == &amdgpu_bo_destroy)
126 return true;
127 return false;
128 }
129
130 /**
131 * amdgpu_bo_placement_from_domain - set buffer's placement
132 * @abo: &amdgpu_bo buffer object whose placement is to be set
133 * @domain: requested domain
134 *
135 * Sets buffer's placement according to requested domain and the buffer's
136 * flags.
137 */
amdgpu_bo_placement_from_domain(struct amdgpu_bo * abo,u32 domain)138 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
139 {
140 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
141 struct ttm_placement *placement = &abo->placement;
142 struct ttm_place *places = abo->placements;
143 u64 flags = abo->flags;
144 u32 c = 0;
145
146 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
147 unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
148
149 places[c].fpfn = 0;
150 places[c].lpfn = 0;
151 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
152 TTM_PL_FLAG_VRAM;
153
154 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
155 places[c].lpfn = visible_pfn;
156 else
157 places[c].flags |= TTM_PL_FLAG_TOPDOWN;
158
159 if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
160 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
161 c++;
162 }
163
164 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
165 places[c].fpfn = 0;
166 if (flags & AMDGPU_GEM_CREATE_SHADOW)
167 places[c].lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
168 else
169 places[c].lpfn = 0;
170 places[c].flags = TTM_PL_FLAG_TT;
171 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
172 places[c].flags |= TTM_PL_FLAG_WC |
173 TTM_PL_FLAG_UNCACHED;
174 else
175 places[c].flags |= TTM_PL_FLAG_CACHED;
176 c++;
177 }
178
179 if (domain & AMDGPU_GEM_DOMAIN_CPU) {
180 places[c].fpfn = 0;
181 places[c].lpfn = 0;
182 places[c].flags = TTM_PL_FLAG_SYSTEM;
183 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
184 places[c].flags |= TTM_PL_FLAG_WC |
185 TTM_PL_FLAG_UNCACHED;
186 else
187 places[c].flags |= TTM_PL_FLAG_CACHED;
188 c++;
189 }
190
191 if (domain & AMDGPU_GEM_DOMAIN_GDS) {
192 places[c].fpfn = 0;
193 places[c].lpfn = 0;
194 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
195 c++;
196 }
197
198 if (domain & AMDGPU_GEM_DOMAIN_GWS) {
199 places[c].fpfn = 0;
200 places[c].lpfn = 0;
201 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
202 c++;
203 }
204
205 if (domain & AMDGPU_GEM_DOMAIN_OA) {
206 places[c].fpfn = 0;
207 places[c].lpfn = 0;
208 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
209 c++;
210 }
211
212 if (!c) {
213 places[c].fpfn = 0;
214 places[c].lpfn = 0;
215 places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
216 c++;
217 }
218
219 BUG_ON(c >= AMDGPU_BO_MAX_PLACEMENTS);
220
221 placement->num_placement = c;
222 placement->placement = places;
223
224 placement->num_busy_placement = c;
225 placement->busy_placement = places;
226 }
227
228 /**
229 * amdgpu_bo_create_reserved - create reserved BO for kernel use
230 *
231 * @adev: amdgpu device object
232 * @size: size for the new BO
233 * @align: alignment for the new BO
234 * @domain: where to place it
235 * @bo_ptr: used to initialize BOs in structures
236 * @gpu_addr: GPU addr of the pinned BO
237 * @cpu_addr: optional CPU address mapping
238 *
239 * Allocates and pins a BO for kernel internal use, and returns it still
240 * reserved.
241 *
242 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
243 *
244 * Returns:
245 * 0 on success, negative error code otherwise.
246 */
amdgpu_bo_create_reserved(struct amdgpu_device * adev,unsigned long size,int align,u32 domain,struct amdgpu_bo ** bo_ptr,u64 * gpu_addr,void ** cpu_addr)247 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
248 unsigned long size, int align,
249 u32 domain, struct amdgpu_bo **bo_ptr,
250 u64 *gpu_addr, void **cpu_addr)
251 {
252 struct amdgpu_bo_param bp;
253 bool free = false;
254 int r;
255
256 memset(&bp, 0, sizeof(bp));
257 bp.size = size;
258 bp.byte_align = align;
259 bp.domain = domain;
260 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
261 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
262 bp.type = ttm_bo_type_kernel;
263 bp.resv = NULL;
264
265 if (!*bo_ptr) {
266 r = amdgpu_bo_create(adev, &bp, bo_ptr);
267 if (r) {
268 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
269 r);
270 return r;
271 }
272 free = true;
273 }
274
275 r = amdgpu_bo_reserve(*bo_ptr, false);
276 if (r) {
277 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
278 goto error_free;
279 }
280
281 r = amdgpu_bo_pin(*bo_ptr, domain);
282 if (r) {
283 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
284 goto error_unreserve;
285 }
286
287 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
288 if (r) {
289 dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
290 goto error_unpin;
291 }
292
293 if (gpu_addr)
294 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
295
296 if (cpu_addr) {
297 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
298 if (r) {
299 dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
300 goto error_unpin;
301 }
302 }
303
304 return 0;
305
306 error_unpin:
307 amdgpu_bo_unpin(*bo_ptr);
308 error_unreserve:
309 amdgpu_bo_unreserve(*bo_ptr);
310
311 error_free:
312 if (free)
313 amdgpu_bo_unref(bo_ptr);
314
315 return r;
316 }
317
318 /**
319 * amdgpu_bo_create_kernel - create BO for kernel use
320 *
321 * @adev: amdgpu device object
322 * @size: size for the new BO
323 * @align: alignment for the new BO
324 * @domain: where to place it
325 * @bo_ptr: used to initialize BOs in structures
326 * @gpu_addr: GPU addr of the pinned BO
327 * @cpu_addr: optional CPU address mapping
328 *
329 * Allocates and pins a BO for kernel internal use.
330 *
331 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
332 *
333 * Returns:
334 * 0 on success, negative error code otherwise.
335 */
amdgpu_bo_create_kernel(struct amdgpu_device * adev,unsigned long size,int align,u32 domain,struct amdgpu_bo ** bo_ptr,u64 * gpu_addr,void ** cpu_addr)336 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
337 unsigned long size, int align,
338 u32 domain, struct amdgpu_bo **bo_ptr,
339 u64 *gpu_addr, void **cpu_addr)
340 {
341 int r;
342
343 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
344 gpu_addr, cpu_addr);
345
346 if (r)
347 return r;
348
349 amdgpu_bo_unreserve(*bo_ptr);
350
351 return 0;
352 }
353
354 /**
355 * amdgpu_bo_free_kernel - free BO for kernel use
356 *
357 * @bo: amdgpu BO to free
358 * @gpu_addr: pointer to where the BO's GPU memory space address was stored
359 * @cpu_addr: pointer to where the BO's CPU memory space address was stored
360 *
361 * unmaps and unpin a BO for kernel internal use.
362 */
amdgpu_bo_free_kernel(struct amdgpu_bo ** bo,u64 * gpu_addr,void ** cpu_addr)363 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
364 void **cpu_addr)
365 {
366 if (*bo == NULL)
367 return;
368
369 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
370 if (cpu_addr)
371 amdgpu_bo_kunmap(*bo);
372
373 amdgpu_bo_unpin(*bo);
374 amdgpu_bo_unreserve(*bo);
375 }
376 amdgpu_bo_unref(bo);
377
378 if (gpu_addr)
379 *gpu_addr = 0;
380
381 if (cpu_addr)
382 *cpu_addr = NULL;
383 }
384
385 /* Validate bo size is bit bigger then the request domain */
amdgpu_bo_validate_size(struct amdgpu_device * adev,unsigned long size,u32 domain)386 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
387 unsigned long size, u32 domain)
388 {
389 struct ttm_mem_type_manager *man = NULL;
390
391 /*
392 * If GTT is part of requested domains the check must succeed to
393 * allow fall back to GTT
394 */
395 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
396 man = &adev->mman.bdev.man[TTM_PL_TT];
397
398 if (size < (man->size << PAGE_SHIFT))
399 return true;
400 else
401 goto fail;
402 }
403
404 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
405 man = &adev->mman.bdev.man[TTM_PL_VRAM];
406
407 if (size < (man->size << PAGE_SHIFT))
408 return true;
409 else
410 goto fail;
411 }
412
413
414 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
415 return true;
416
417 fail:
418 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
419 man->size << PAGE_SHIFT);
420 return false;
421 }
422
amdgpu_bo_do_create(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo ** bo_ptr)423 static int amdgpu_bo_do_create(struct amdgpu_device *adev,
424 struct amdgpu_bo_param *bp,
425 struct amdgpu_bo **bo_ptr)
426 {
427 struct ttm_operation_ctx ctx = {
428 .interruptible = (bp->type != ttm_bo_type_kernel),
429 .no_wait_gpu = false,
430 .resv = bp->resv,
431 .flags = TTM_OPT_FLAG_ALLOW_RES_EVICT
432 };
433 struct amdgpu_bo *bo;
434 unsigned long page_align, size = bp->size;
435 size_t acc_size;
436 int r;
437
438 page_align = roundup(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
439 size = ALIGN(size, PAGE_SIZE);
440
441 if (!amdgpu_bo_validate_size(adev, size, bp->domain))
442 return -ENOMEM;
443
444 *bo_ptr = NULL;
445
446 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
447 sizeof(struct amdgpu_bo));
448
449 bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
450 if (bo == NULL)
451 return -ENOMEM;
452 drm_gem_private_object_init(adev->ddev, &bo->gem_base, size);
453 INIT_LIST_HEAD(&bo->shadow_list);
454 INIT_LIST_HEAD(&bo->va);
455 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
456 bp->domain;
457 bo->allowed_domains = bo->preferred_domains;
458 if (bp->type != ttm_bo_type_kernel &&
459 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
460 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
461
462 bo->flags = bp->flags;
463
464 #ifdef CONFIG_X86_32
465 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
466 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
467 */
468 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
469 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
470 /* Don't try to enable write-combining when it can't work, or things
471 * may be slow
472 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
473 */
474
475 #ifndef CONFIG_COMPILE_TEST
476 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
477 thanks to write-combining
478 #endif
479
480 if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
481 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
482 "better performance thanks to write-combining\n");
483 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
484 #else
485 /* For architectures that don't support WC memory,
486 * mask out the WC flag from the BO
487 */
488 if (!drm_arch_can_wc_memory())
489 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
490 #endif
491
492 bo->tbo.bdev = &adev->mman.bdev;
493 amdgpu_bo_placement_from_domain(bo, bp->domain);
494 if (bp->type == ttm_bo_type_kernel)
495 bo->tbo.priority = 1;
496
497 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,
498 &bo->placement, page_align, &ctx, acc_size,
499 NULL, bp->resv, &amdgpu_bo_destroy);
500 if (unlikely(r != 0))
501 return r;
502
503 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
504 bo->tbo.mem.mem_type == TTM_PL_VRAM &&
505 bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT)
506 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
507 ctx.bytes_moved);
508 else
509 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
510
511 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
512 bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
513 struct dma_fence *fence;
514
515 r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
516 if (unlikely(r))
517 goto fail_unreserve;
518
519 amdgpu_bo_fence(bo, fence, false);
520 dma_fence_put(bo->tbo.moving);
521 bo->tbo.moving = dma_fence_get(fence);
522 dma_fence_put(fence);
523 }
524 if (!bp->resv)
525 amdgpu_bo_unreserve(bo);
526 *bo_ptr = bo;
527
528 trace_amdgpu_bo_create(bo);
529
530 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
531 if (bp->type == ttm_bo_type_device)
532 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
533
534 return 0;
535
536 fail_unreserve:
537 if (!bp->resv)
538 ww_mutex_unlock(&bo->tbo.resv->lock);
539 amdgpu_bo_unref(&bo);
540 return r;
541 }
542
amdgpu_bo_create_shadow(struct amdgpu_device * adev,unsigned long size,int byte_align,struct amdgpu_bo * bo)543 static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
544 unsigned long size, int byte_align,
545 struct amdgpu_bo *bo)
546 {
547 struct amdgpu_bo_param bp;
548 int r;
549
550 if (bo->shadow)
551 return 0;
552
553 memset(&bp, 0, sizeof(bp));
554 bp.size = size;
555 bp.byte_align = byte_align;
556 bp.domain = AMDGPU_GEM_DOMAIN_GTT;
557 bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
558 AMDGPU_GEM_CREATE_SHADOW;
559 bp.type = ttm_bo_type_kernel;
560 bp.resv = bo->tbo.resv;
561
562 r = amdgpu_bo_do_create(adev, &bp, &bo->shadow);
563 if (!r) {
564 bo->shadow->parent = amdgpu_bo_ref(bo);
565 mutex_lock(&adev->shadow_list_lock);
566 list_add_tail(&bo->shadow_list, &adev->shadow_list);
567 mutex_unlock(&adev->shadow_list_lock);
568 }
569
570 return r;
571 }
572
573 /**
574 * amdgpu_bo_create - create an &amdgpu_bo buffer object
575 * @adev: amdgpu device object
576 * @bp: parameters to be used for the buffer object
577 * @bo_ptr: pointer to the buffer object pointer
578 *
579 * Creates an &amdgpu_bo buffer object; and if requested, also creates a
580 * shadow object.
581 * Shadow object is used to backup the original buffer object, and is always
582 * in GTT.
583 *
584 * Returns:
585 * 0 for success or a negative error code on failure.
586 */
amdgpu_bo_create(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo ** bo_ptr)587 int amdgpu_bo_create(struct amdgpu_device *adev,
588 struct amdgpu_bo_param *bp,
589 struct amdgpu_bo **bo_ptr)
590 {
591 u64 flags = bp->flags;
592 int r;
593
594 bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW;
595 r = amdgpu_bo_do_create(adev, bp, bo_ptr);
596 if (r)
597 return r;
598
599 if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_bo_need_backup(adev)) {
600 if (!bp->resv)
601 WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
602 NULL));
603
604 r = amdgpu_bo_create_shadow(adev, bp->size, bp->byte_align, (*bo_ptr));
605
606 if (!bp->resv)
607 reservation_object_unlock((*bo_ptr)->tbo.resv);
608
609 if (r)
610 amdgpu_bo_unref(bo_ptr);
611 }
612
613 return r;
614 }
615
616 /**
617 * amdgpu_bo_backup_to_shadow - Backs up an &amdgpu_bo buffer object
618 * @adev: amdgpu device object
619 * @ring: amdgpu_ring for the engine handling the buffer operations
620 * @bo: &amdgpu_bo buffer to be backed up
621 * @resv: reservation object with embedded fence
622 * @fence: dma_fence associated with the operation
623 * @direct: whether to submit the job directly
624 *
625 * Copies an &amdgpu_bo buffer object to its shadow object.
626 * Not used for now.
627 *
628 * Returns:
629 * 0 for success or a negative error code on failure.
630 */
amdgpu_bo_backup_to_shadow(struct amdgpu_device * adev,struct amdgpu_ring * ring,struct amdgpu_bo * bo,struct reservation_object * resv,struct dma_fence ** fence,bool direct)631 int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
632 struct amdgpu_ring *ring,
633 struct amdgpu_bo *bo,
634 struct reservation_object *resv,
635 struct dma_fence **fence,
636 bool direct)
637
638 {
639 struct amdgpu_bo *shadow = bo->shadow;
640 uint64_t bo_addr, shadow_addr;
641 int r;
642
643 if (!shadow)
644 return -EINVAL;
645
646 bo_addr = amdgpu_bo_gpu_offset(bo);
647 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
648
649 r = reservation_object_reserve_shared(bo->tbo.resv);
650 if (r)
651 goto err;
652
653 r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
654 amdgpu_bo_size(bo), resv, fence,
655 direct, false);
656 if (!r)
657 amdgpu_bo_fence(bo, *fence, true);
658
659 err:
660 return r;
661 }
662
663 /**
664 * amdgpu_bo_validate - validate an &amdgpu_bo buffer object
665 * @bo: pointer to the buffer object
666 *
667 * Sets placement according to domain; and changes placement and caching
668 * policy of the buffer object according to the placement.
669 * This is used for validating shadow bos. It calls ttm_bo_validate() to
670 * make sure the buffer is resident where it needs to be.
671 *
672 * Returns:
673 * 0 for success or a negative error code on failure.
674 */
amdgpu_bo_validate(struct amdgpu_bo * bo)675 int amdgpu_bo_validate(struct amdgpu_bo *bo)
676 {
677 struct ttm_operation_ctx ctx = { false, false };
678 uint32_t domain;
679 int r;
680
681 if (bo->pin_count)
682 return 0;
683
684 domain = bo->preferred_domains;
685
686 retry:
687 amdgpu_bo_placement_from_domain(bo, domain);
688 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
689 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
690 domain = bo->allowed_domains;
691 goto retry;
692 }
693
694 return r;
695 }
696
697 /**
698 * amdgpu_bo_restore_from_shadow - restore an &amdgpu_bo buffer object
699 * @adev: amdgpu device object
700 * @ring: amdgpu_ring for the engine handling the buffer operations
701 * @bo: &amdgpu_bo buffer to be restored
702 * @resv: reservation object with embedded fence
703 * @fence: dma_fence associated with the operation
704 * @direct: whether to submit the job directly
705 *
706 * Copies a buffer object's shadow content back to the object.
707 * This is used for recovering a buffer from its shadow in case of a gpu
708 * reset where vram context may be lost.
709 *
710 * Returns:
711 * 0 for success or a negative error code on failure.
712 */
amdgpu_bo_restore_from_shadow(struct amdgpu_device * adev,struct amdgpu_ring * ring,struct amdgpu_bo * bo,struct reservation_object * resv,struct dma_fence ** fence,bool direct)713 int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
714 struct amdgpu_ring *ring,
715 struct amdgpu_bo *bo,
716 struct reservation_object *resv,
717 struct dma_fence **fence,
718 bool direct)
719
720 {
721 struct amdgpu_bo *shadow = bo->shadow;
722 uint64_t bo_addr, shadow_addr;
723 int r;
724
725 if (!shadow)
726 return -EINVAL;
727
728 bo_addr = amdgpu_bo_gpu_offset(bo);
729 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
730
731 r = reservation_object_reserve_shared(bo->tbo.resv);
732 if (r)
733 goto err;
734
735 r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
736 amdgpu_bo_size(bo), resv, fence,
737 direct, false);
738 if (!r)
739 amdgpu_bo_fence(bo, *fence, true);
740
741 err:
742 return r;
743 }
744
745 /**
746 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
747 * @bo: &amdgpu_bo buffer object to be mapped
748 * @ptr: kernel virtual address to be returned
749 *
750 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
751 * amdgpu_bo_kptr() to get the kernel virtual address.
752 *
753 * Returns:
754 * 0 for success or a negative error code on failure.
755 */
amdgpu_bo_kmap(struct amdgpu_bo * bo,void ** ptr)756 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
757 {
758 void *kptr;
759 long r;
760
761 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
762 return -EPERM;
763
764 kptr = amdgpu_bo_kptr(bo);
765 if (kptr) {
766 if (ptr)
767 *ptr = kptr;
768 return 0;
769 }
770
771 r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
772 MAX_SCHEDULE_TIMEOUT);
773 if (r < 0)
774 return r;
775
776 r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
777 if (r)
778 return r;
779
780 if (ptr)
781 *ptr = amdgpu_bo_kptr(bo);
782
783 return 0;
784 }
785
786 /**
787 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
788 * @bo: &amdgpu_bo buffer object
789 *
790 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
791 *
792 * Returns:
793 * the virtual address of a buffer object area.
794 */
amdgpu_bo_kptr(struct amdgpu_bo * bo)795 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
796 {
797 bool is_iomem;
798
799 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
800 }
801
802 /**
803 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
804 * @bo: &amdgpu_bo buffer object to be unmapped
805 *
806 * Unmaps a kernel map set up by amdgpu_bo_kmap().
807 */
amdgpu_bo_kunmap(struct amdgpu_bo * bo)808 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
809 {
810 if (bo->kmap.bo)
811 ttm_bo_kunmap(&bo->kmap);
812 }
813
814 /**
815 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
816 * @bo: &amdgpu_bo buffer object
817 *
818 * References the contained &ttm_buffer_object.
819 *
820 * Returns:
821 * a refcounted pointer to the &amdgpu_bo buffer object.
822 */
amdgpu_bo_ref(struct amdgpu_bo * bo)823 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
824 {
825 if (bo == NULL)
826 return NULL;
827
828 ttm_bo_get(&bo->tbo);
829 return bo;
830 }
831
832 /**
833 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
834 * @bo: &amdgpu_bo buffer object
835 *
836 * Unreferences the contained &ttm_buffer_object and clear the pointer
837 */
amdgpu_bo_unref(struct amdgpu_bo ** bo)838 void amdgpu_bo_unref(struct amdgpu_bo **bo)
839 {
840 struct ttm_buffer_object *tbo;
841
842 if ((*bo) == NULL)
843 return;
844
845 tbo = &((*bo)->tbo);
846 ttm_bo_put(tbo);
847 *bo = NULL;
848 }
849
850 /**
851 * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object
852 * @bo: &amdgpu_bo buffer object to be pinned
853 * @domain: domain to be pinned to
854 * @min_offset: the start of requested address range
855 * @max_offset: the end of requested address range
856 *
857 * Pins the buffer object according to requested domain and address range. If
858 * the memory is unbound gart memory, binds the pages into gart table. Adjusts
859 * pin_count and pin_size accordingly.
860 *
861 * Pinning means to lock pages in memory along with keeping them at a fixed
862 * offset. It is required when a buffer can not be moved, for example, when
863 * a display buffer is being scanned out.
864 *
865 * Compared with amdgpu_bo_pin(), this function gives more flexibility on
866 * where to pin a buffer if there are specific restrictions on where a buffer
867 * must be located.
868 *
869 * Returns:
870 * 0 for success or a negative error code on failure.
871 */
amdgpu_bo_pin_restricted(struct amdgpu_bo * bo,u32 domain,u64 min_offset,u64 max_offset)872 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
873 u64 min_offset, u64 max_offset)
874 {
875 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
876 struct ttm_operation_ctx ctx = { false, false };
877 int r, i;
878
879 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
880 return -EPERM;
881
882 if (WARN_ON_ONCE(min_offset > max_offset))
883 return -EINVAL;
884
885 /* A shared bo cannot be migrated to VRAM */
886 if (bo->prime_shared_count) {
887 if (domain & AMDGPU_GEM_DOMAIN_GTT)
888 domain = AMDGPU_GEM_DOMAIN_GTT;
889 else
890 return -EINVAL;
891 }
892
893 /* This assumes only APU display buffers are pinned with (VRAM|GTT).
894 * See function amdgpu_display_supported_domains()
895 */
896 domain = amdgpu_bo_get_preferred_pin_domain(adev, domain);
897
898 if (bo->pin_count) {
899 uint32_t mem_type = bo->tbo.mem.mem_type;
900
901 if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
902 return -EINVAL;
903
904 bo->pin_count++;
905
906 if (max_offset != 0) {
907 u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
908 WARN_ON_ONCE(max_offset <
909 (amdgpu_bo_gpu_offset(bo) - domain_start));
910 }
911
912 return 0;
913 }
914
915 bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
916 /* force to pin into visible video ram */
917 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
918 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
919 amdgpu_bo_placement_from_domain(bo, domain);
920 for (i = 0; i < bo->placement.num_placement; i++) {
921 unsigned fpfn, lpfn;
922
923 fpfn = min_offset >> PAGE_SHIFT;
924 lpfn = max_offset >> PAGE_SHIFT;
925
926 if (fpfn > bo->placements[i].fpfn)
927 bo->placements[i].fpfn = fpfn;
928 if (!bo->placements[i].lpfn ||
929 (lpfn && lpfn < bo->placements[i].lpfn))
930 bo->placements[i].lpfn = lpfn;
931 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
932 }
933
934 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
935 if (unlikely(r)) {
936 dev_err(adev->dev, "%p pin failed\n", bo);
937 goto error;
938 }
939
940 bo->pin_count = 1;
941
942 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
943 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
944 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
945 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
946 &adev->visible_pin_size);
947 } else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
948 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
949 }
950
951 error:
952 return r;
953 }
954
955 /**
956 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
957 * @bo: &amdgpu_bo buffer object to be pinned
958 * @domain: domain to be pinned to
959 *
960 * A simple wrapper to amdgpu_bo_pin_restricted().
961 * Provides a simpler API for buffers that do not have any strict restrictions
962 * on where a buffer must be located.
963 *
964 * Returns:
965 * 0 for success or a negative error code on failure.
966 */
amdgpu_bo_pin(struct amdgpu_bo * bo,u32 domain)967 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
968 {
969 return amdgpu_bo_pin_restricted(bo, domain, 0, 0);
970 }
971
972 /**
973 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
974 * @bo: &amdgpu_bo buffer object to be unpinned
975 *
976 * Decreases the pin_count, and clears the flags if pin_count reaches 0.
977 * Changes placement and pin size accordingly.
978 *
979 * Returns:
980 * 0 for success or a negative error code on failure.
981 */
amdgpu_bo_unpin(struct amdgpu_bo * bo)982 int amdgpu_bo_unpin(struct amdgpu_bo *bo)
983 {
984 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
985 struct ttm_operation_ctx ctx = { false, false };
986 int r, i;
987
988 if (!bo->pin_count) {
989 dev_warn(adev->dev, "%p unpin not necessary\n", bo);
990 return 0;
991 }
992 bo->pin_count--;
993 if (bo->pin_count)
994 return 0;
995
996 amdgpu_bo_subtract_pin_size(bo);
997
998 for (i = 0; i < bo->placement.num_placement; i++) {
999 bo->placements[i].lpfn = 0;
1000 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
1001 }
1002 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1003 if (unlikely(r))
1004 dev_err(adev->dev, "%p validate failed for unpin\n", bo);
1005
1006 return r;
1007 }
1008
1009 /**
1010 * amdgpu_bo_evict_vram - evict VRAM buffers
1011 * @adev: amdgpu device object
1012 *
1013 * Evicts all VRAM buffers on the lru list of the memory type.
1014 * Mainly used for evicting vram at suspend time.
1015 *
1016 * Returns:
1017 * 0 for success or a negative error code on failure.
1018 */
amdgpu_bo_evict_vram(struct amdgpu_device * adev)1019 int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
1020 {
1021 /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
1022 if (0 && (adev->flags & AMD_IS_APU)) {
1023 /* Useless to evict on IGP chips */
1024 return 0;
1025 }
1026 return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
1027 }
1028
1029 static const char *amdgpu_vram_names[] = {
1030 "UNKNOWN",
1031 "GDDR1",
1032 "DDR2",
1033 "GDDR3",
1034 "GDDR4",
1035 "GDDR5",
1036 "HBM",
1037 "DDR3",
1038 "DDR4",
1039 };
1040
1041 /**
1042 * amdgpu_bo_init - initialize memory manager
1043 * @adev: amdgpu device object
1044 *
1045 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1046 *
1047 * Returns:
1048 * 0 for success or a negative error code on failure.
1049 */
amdgpu_bo_init(struct amdgpu_device * adev)1050 int amdgpu_bo_init(struct amdgpu_device *adev)
1051 {
1052 /* reserve PAT memory space to WC for VRAM */
1053 arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1054 adev->gmc.aper_size);
1055
1056 /* Add an MTRR for the VRAM */
1057 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1058 adev->gmc.aper_size);
1059 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
1060 adev->gmc.mc_vram_size >> 20,
1061 (unsigned long long)adev->gmc.aper_size >> 20);
1062 DRM_INFO("RAM width %dbits %s\n",
1063 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1064 return amdgpu_ttm_init(adev);
1065 }
1066
1067 /**
1068 * amdgpu_bo_late_init - late init
1069 * @adev: amdgpu device object
1070 *
1071 * Calls amdgpu_ttm_late_init() to free resources used earlier during
1072 * initialization.
1073 *
1074 * Returns:
1075 * 0 for success or a negative error code on failure.
1076 */
amdgpu_bo_late_init(struct amdgpu_device * adev)1077 int amdgpu_bo_late_init(struct amdgpu_device *adev)
1078 {
1079 amdgpu_ttm_late_init(adev);
1080
1081 return 0;
1082 }
1083
1084 /**
1085 * amdgpu_bo_fini - tear down memory manager
1086 * @adev: amdgpu device object
1087 *
1088 * Reverses amdgpu_bo_init() to tear down memory manager.
1089 */
amdgpu_bo_fini(struct amdgpu_device * adev)1090 void amdgpu_bo_fini(struct amdgpu_device *adev)
1091 {
1092 amdgpu_ttm_fini(adev);
1093 arch_phys_wc_del(adev->gmc.vram_mtrr);
1094 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
1095 }
1096
1097 /**
1098 * amdgpu_bo_fbdev_mmap - mmap fbdev memory
1099 * @bo: &amdgpu_bo buffer object
1100 * @vma: vma as input from the fbdev mmap method
1101 *
1102 * Calls ttm_fbdev_mmap() to mmap fbdev memory if it is backed by a bo.
1103 *
1104 * Returns:
1105 * 0 for success or a negative error code on failure.
1106 */
amdgpu_bo_fbdev_mmap(struct amdgpu_bo * bo,struct vm_area_struct * vma)1107 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
1108 struct vm_area_struct *vma)
1109 {
1110 return ttm_fbdev_mmap(vma, &bo->tbo);
1111 }
1112
1113 /**
1114 * amdgpu_bo_set_tiling_flags - set tiling flags
1115 * @bo: &amdgpu_bo buffer object
1116 * @tiling_flags: new flags
1117 *
1118 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1119 * kernel driver to set the tiling flags on a buffer.
1120 *
1121 * Returns:
1122 * 0 for success or a negative error code on failure.
1123 */
amdgpu_bo_set_tiling_flags(struct amdgpu_bo * bo,u64 tiling_flags)1124 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1125 {
1126 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1127
1128 if (adev->family <= AMDGPU_FAMILY_CZ &&
1129 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1130 return -EINVAL;
1131
1132 bo->tiling_flags = tiling_flags;
1133 return 0;
1134 }
1135
1136 /**
1137 * amdgpu_bo_get_tiling_flags - get tiling flags
1138 * @bo: &amdgpu_bo buffer object
1139 * @tiling_flags: returned flags
1140 *
1141 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1142 * set the tiling flags on a buffer.
1143 */
amdgpu_bo_get_tiling_flags(struct amdgpu_bo * bo,u64 * tiling_flags)1144 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1145 {
1146 lockdep_assert_held(&bo->tbo.resv->lock.base);
1147
1148 if (tiling_flags)
1149 *tiling_flags = bo->tiling_flags;
1150 }
1151
1152 /**
1153 * amdgpu_bo_set_metadata - set metadata
1154 * @bo: &amdgpu_bo buffer object
1155 * @metadata: new metadata
1156 * @metadata_size: size of the new metadata
1157 * @flags: flags of the new metadata
1158 *
1159 * Sets buffer object's metadata, its size and flags.
1160 * Used via GEM ioctl.
1161 *
1162 * Returns:
1163 * 0 for success or a negative error code on failure.
1164 */
amdgpu_bo_set_metadata(struct amdgpu_bo * bo,void * metadata,uint32_t metadata_size,uint64_t flags)1165 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
1166 uint32_t metadata_size, uint64_t flags)
1167 {
1168 void *buffer;
1169
1170 if (!metadata_size) {
1171 if (bo->metadata_size) {
1172 kfree(bo->metadata);
1173 bo->metadata = NULL;
1174 bo->metadata_size = 0;
1175 }
1176 return 0;
1177 }
1178
1179 if (metadata == NULL)
1180 return -EINVAL;
1181
1182 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1183 if (buffer == NULL)
1184 return -ENOMEM;
1185
1186 kfree(bo->metadata);
1187 bo->metadata_flags = flags;
1188 bo->metadata = buffer;
1189 bo->metadata_size = metadata_size;
1190
1191 return 0;
1192 }
1193
1194 /**
1195 * amdgpu_bo_get_metadata - get metadata
1196 * @bo: &amdgpu_bo buffer object
1197 * @buffer: returned metadata
1198 * @buffer_size: size of the buffer
1199 * @metadata_size: size of the returned metadata
1200 * @flags: flags of the returned metadata
1201 *
1202 * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1203 * less than metadata_size.
1204 * Used via GEM ioctl.
1205 *
1206 * Returns:
1207 * 0 for success or a negative error code on failure.
1208 */
amdgpu_bo_get_metadata(struct amdgpu_bo * bo,void * buffer,size_t buffer_size,uint32_t * metadata_size,uint64_t * flags)1209 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1210 size_t buffer_size, uint32_t *metadata_size,
1211 uint64_t *flags)
1212 {
1213 if (!buffer && !metadata_size)
1214 return -EINVAL;
1215
1216 if (buffer) {
1217 if (buffer_size < bo->metadata_size)
1218 return -EINVAL;
1219
1220 if (bo->metadata_size)
1221 memcpy(buffer, bo->metadata, bo->metadata_size);
1222 }
1223
1224 if (metadata_size)
1225 *metadata_size = bo->metadata_size;
1226 if (flags)
1227 *flags = bo->metadata_flags;
1228
1229 return 0;
1230 }
1231
1232 /**
1233 * amdgpu_bo_move_notify - notification about a memory move
1234 * @bo: pointer to a buffer object
1235 * @evict: if this move is evicting the buffer from the graphics address space
1236 * @new_mem: new information of the bufer object
1237 *
1238 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1239 * bookkeeping.
1240 * TTM driver callback which is called when ttm moves a buffer.
1241 */
amdgpu_bo_move_notify(struct ttm_buffer_object * bo,bool evict,struct ttm_mem_reg * new_mem)1242 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1243 bool evict,
1244 struct ttm_mem_reg *new_mem)
1245 {
1246 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1247 struct amdgpu_bo *abo;
1248 struct ttm_mem_reg *old_mem = &bo->mem;
1249
1250 if (!amdgpu_bo_is_amdgpu_bo(bo))
1251 return;
1252
1253 abo = ttm_to_amdgpu_bo(bo);
1254 amdgpu_vm_bo_invalidate(adev, abo, evict);
1255
1256 amdgpu_bo_kunmap(abo);
1257
1258 /* remember the eviction */
1259 if (evict)
1260 atomic64_inc(&adev->num_evictions);
1261
1262 /* update statistics */
1263 if (!new_mem)
1264 return;
1265
1266 /* move_notify is called before move happens */
1267 trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
1268 }
1269
1270 /**
1271 * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1272 * @bo: pointer to a buffer object
1273 *
1274 * Notifies the driver we are taking a fault on this BO and have reserved it,
1275 * also performs bookkeeping.
1276 * TTM driver callback for dealing with vm faults.
1277 *
1278 * Returns:
1279 * 0 for success or a negative error code on failure.
1280 */
amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object * bo)1281 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1282 {
1283 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1284 struct ttm_operation_ctx ctx = { false, false };
1285 struct amdgpu_bo *abo;
1286 unsigned long offset, size;
1287 int r;
1288
1289 if (!amdgpu_bo_is_amdgpu_bo(bo))
1290 return 0;
1291
1292 abo = ttm_to_amdgpu_bo(bo);
1293
1294 /* Remember that this BO was accessed by the CPU */
1295 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1296
1297 if (bo->mem.mem_type != TTM_PL_VRAM)
1298 return 0;
1299
1300 size = bo->mem.num_pages << PAGE_SHIFT;
1301 offset = bo->mem.start << PAGE_SHIFT;
1302 if ((offset + size) <= adev->gmc.visible_vram_size)
1303 return 0;
1304
1305 /* Can't move a pinned BO to visible VRAM */
1306 if (abo->pin_count > 0)
1307 return -EINVAL;
1308
1309 /* hurrah the memory is not visible ! */
1310 atomic64_inc(&adev->num_vram_cpu_page_faults);
1311 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1312 AMDGPU_GEM_DOMAIN_GTT);
1313
1314 /* Avoid costly evictions; only set GTT as a busy placement */
1315 abo->placement.num_busy_placement = 1;
1316 abo->placement.busy_placement = &abo->placements[1];
1317
1318 r = ttm_bo_validate(bo, &abo->placement, &ctx);
1319 if (unlikely(r != 0))
1320 return r;
1321
1322 offset = bo->mem.start << PAGE_SHIFT;
1323 /* this should never happen */
1324 if (bo->mem.mem_type == TTM_PL_VRAM &&
1325 (offset + size) > adev->gmc.visible_vram_size)
1326 return -EINVAL;
1327
1328 return 0;
1329 }
1330
1331 /**
1332 * amdgpu_bo_fence - add fence to buffer object
1333 *
1334 * @bo: buffer object in question
1335 * @fence: fence to add
1336 * @shared: true if fence should be added shared
1337 *
1338 */
amdgpu_bo_fence(struct amdgpu_bo * bo,struct dma_fence * fence,bool shared)1339 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1340 bool shared)
1341 {
1342 struct reservation_object *resv = bo->tbo.resv;
1343
1344 if (shared)
1345 reservation_object_add_shared_fence(resv, fence);
1346 else
1347 reservation_object_add_excl_fence(resv, fence);
1348 }
1349
1350 /**
1351 * amdgpu_bo_gpu_offset - return GPU offset of bo
1352 * @bo: amdgpu object for which we query the offset
1353 *
1354 * Note: object should either be pinned or reserved when calling this
1355 * function, it might be useful to add check for this for debugging.
1356 *
1357 * Returns:
1358 * current GPU offset of the object.
1359 */
amdgpu_bo_gpu_offset(struct amdgpu_bo * bo)1360 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1361 {
1362 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
1363 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
1364 !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem));
1365 WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
1366 !bo->pin_count);
1367 WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
1368 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
1369 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1370
1371 return bo->tbo.offset;
1372 }
1373
1374 /**
1375 * amdgpu_bo_get_preferred_pin_domain - get preferred domain for scanout
1376 * @adev: amdgpu device object
1377 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1378 *
1379 * Returns:
1380 * Which of the allowed domains is preferred for pinning the BO for scanout.
1381 */
amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device * adev,uint32_t domain)1382 uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev,
1383 uint32_t domain)
1384 {
1385 if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
1386 domain = AMDGPU_GEM_DOMAIN_VRAM;
1387 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1388 domain = AMDGPU_GEM_DOMAIN_GTT;
1389 }
1390 return domain;
1391 }
1392
