Lines Matching refs:eb
307 static int eb_parse(struct i915_execbuffer *eb);
308 static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb,
310 static void eb_unpin_engine(struct i915_execbuffer *eb);
312 static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) in eb_use_cmdparser() argument
314 return intel_engine_requires_cmd_parser(eb->engine) || in eb_use_cmdparser()
315 (intel_engine_using_cmd_parser(eb->engine) && in eb_use_cmdparser()
316 eb->args->batch_len); in eb_use_cmdparser()
319 static int eb_create(struct i915_execbuffer *eb) in eb_create() argument
321 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { in eb_create()
322 unsigned int size = 1 + ilog2(eb->buffer_count); in eb_create()
348 eb->buckets = kzalloc(sizeof(struct hlist_head) << size, in eb_create()
350 if (eb->buckets) in eb_create()
357 eb->lut_size = size; in eb_create()
359 eb->lut_size = -eb->buffer_count; in eb_create()
422 eb_pin_vma(struct i915_execbuffer *eb, in eb_pin_vma() argument
440 if (unlikely(i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags))) { in eb_pin_vma()
445 if (unlikely(i915_vma_pin_ww(vma, &eb->ww, in eb_pin_vma()
481 eb_validate_vma(struct i915_execbuffer *eb, in eb_validate_vma() argument
485 if (unlikely(entry->flags & eb->invalid_flags)) in eb_validate_vma()
514 if (!eb->reloc_cache.has_fence) { in eb_validate_vma()
518 eb->reloc_cache.needs_unfenced) && in eb_validate_vma()
524 entry->flags |= eb->context_flags; in eb_validate_vma()
530 eb_add_vma(struct i915_execbuffer *eb, in eb_add_vma() argument
534 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_add_vma()
535 struct eb_vma *ev = &eb->vma[i]; in eb_add_vma()
543 if (eb->lut_size > 0) { in eb_add_vma()
546 &eb->buckets[hash_32(entry->handle, in eb_add_vma()
547 eb->lut_size)]); in eb_add_vma()
551 list_add_tail(&ev->reloc_link, &eb->relocs); in eb_add_vma()
566 if (eb->reloc_cache.has_fence) in eb_add_vma()
569 eb->batch = ev; in eb_add_vma()
590 static int eb_reserve_vma(struct i915_execbuffer *eb, in eb_reserve_vma() argument
605 err = i915_vma_pin_ww(vma, &eb->ww, in eb_reserve_vma()
613 eb->args->flags |= __EXEC_HAS_RELOC; in eb_reserve_vma()
633 static int eb_reserve(struct i915_execbuffer *eb) in eb_reserve() argument
635 const unsigned int count = eb->buffer_count; in eb_reserve()
657 list_for_each_entry(ev, &eb->unbound, bind_link) { in eb_reserve()
658 err = eb_reserve_vma(eb, ev, pin_flags); in eb_reserve()
666 INIT_LIST_HEAD(&eb->unbound); in eb_reserve()
671 ev = &eb->vma[i]; in eb_reserve()
681 list_add(&ev->bind_link, &eb->unbound); in eb_reserve()
684 list_add_tail(&ev->bind_link, &eb->unbound); in eb_reserve()
691 list_splice_tail(&last, &eb->unbound); in eb_reserve()
699 mutex_lock(&eb->context->vm->mutex); in eb_reserve()
700 err = i915_gem_evict_vm(eb->context->vm); in eb_reserve()
701 mutex_unlock(&eb->context->vm->mutex); in eb_reserve()
714 static unsigned int eb_batch_index(const struct i915_execbuffer *eb) in eb_batch_index() argument
716 if (eb->args->flags & I915_EXEC_BATCH_FIRST) in eb_batch_index()
719 return eb->buffer_count - 1; in eb_batch_index()
722 static int eb_select_context(struct i915_execbuffer *eb) in eb_select_context() argument
726 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); in eb_select_context()
730 eb->gem_context = ctx; in eb_select_context()
732 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; in eb_select_context()
734 eb->context_flags = 0; in eb_select_context()
736 eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS; in eb_select_context()
741 static int __eb_add_lut(struct i915_execbuffer *eb, in __eb_add_lut() argument
744 struct i915_gem_context *ctx = eb->gem_context; in __eb_add_lut()
773 if (idr_find(&eb->file->object_idr, handle) == obj) { in __eb_add_lut()
795 static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) in eb_lookup_vma() argument
797 struct i915_address_space *vm = eb->context->vm; in eb_lookup_vma()
805 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); in eb_lookup_vma()
812 obj = i915_gem_object_lookup(eb->file, handle); in eb_lookup_vma()
822 err = __eb_add_lut(eb, handle, vma); in eb_lookup_vma()
832 static int eb_lookup_vmas(struct i915_execbuffer *eb) in eb_lookup_vmas() argument
834 struct drm_i915_private *i915 = eb->i915; in eb_lookup_vmas()
835 unsigned int batch = eb_batch_index(eb); in eb_lookup_vmas()
839 INIT_LIST_HEAD(&eb->relocs); in eb_lookup_vmas()
841 for (i = 0; i < eb->buffer_count; i++) { in eb_lookup_vmas()
844 vma = eb_lookup_vma(eb, eb->exec[i].handle); in eb_lookup_vmas()
850 err = eb_validate_vma(eb, &eb->exec[i], vma); in eb_lookup_vmas()
856 eb_add_vma(eb, i, batch, vma); in eb_lookup_vmas()
859 if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)) { in eb_lookup_vmas()
866 eb->batch_start_offset, eb->batch_len, in eb_lookup_vmas()
867 eb->batch->vma->size)) { in eb_lookup_vmas()
872 if (eb->batch_len == 0) in eb_lookup_vmas()
873 eb->batch_len = eb->batch->vma->size - eb->batch_start_offset; in eb_lookup_vmas()
874 if (unlikely(eb->batch_len == 0)) { /* impossible! */ in eb_lookup_vmas()
882 eb->vma[i].vma = NULL; in eb_lookup_vmas()
886 static int eb_validate_vmas(struct i915_execbuffer *eb) in eb_validate_vmas() argument
891 INIT_LIST_HEAD(&eb->unbound); in eb_validate_vmas()
893 for (i = 0; i < eb->buffer_count; i++) { in eb_validate_vmas()
894 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_validate_vmas()
895 struct eb_vma *ev = &eb->vma[i]; in eb_validate_vmas()
898 err = i915_gem_object_lock(vma->obj, &eb->ww); in eb_validate_vmas()
902 if (eb_pin_vma(eb, entry, ev)) { in eb_validate_vmas()
905 eb->args->flags |= __EXEC_HAS_RELOC; in eb_validate_vmas()
910 list_add_tail(&ev->bind_link, &eb->unbound); in eb_validate_vmas()
919 eb_vma_misplaced(&eb->exec[i], vma, ev->flags)); in eb_validate_vmas()
922 if (!list_empty(&eb->unbound)) in eb_validate_vmas()
923 return eb_reserve(eb); in eb_validate_vmas()
929 eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) in eb_get_vma() argument
931 if (eb->lut_size < 0) { in eb_get_vma()
932 if (handle >= -eb->lut_size) in eb_get_vma()
934 return &eb->vma[handle]; in eb_get_vma()
939 head = &eb->buckets[hash_32(handle, eb->lut_size)]; in eb_get_vma()
948 static void eb_release_vmas(struct i915_execbuffer *eb, bool final) in eb_release_vmas() argument
950 const unsigned int count = eb->buffer_count; in eb_release_vmas()
954 struct eb_vma *ev = &eb->vma[i]; in eb_release_vmas()
966 eb_unpin_engine(eb); in eb_release_vmas()
969 static void eb_destroy(const struct i915_execbuffer *eb) in eb_destroy() argument
971 GEM_BUG_ON(eb->reloc_cache.rq); in eb_destroy()
973 if (eb->lut_size > 0) in eb_destroy()
974 kfree(eb->buckets); in eb_destroy()
1026 static void reloc_cache_put_pool(struct i915_execbuffer *eb, struct reloc_cache *cache) in reloc_cache_put_pool() argument
1042 static void reloc_gpu_flush(struct i915_execbuffer *eb, struct reloc_cache *cache) in reloc_gpu_flush() argument
1055 reloc_cache_put_pool(eb, cache); in reloc_gpu_flush()
1058 eb->reloc_pool = NULL; in reloc_gpu_flush()
1061 static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb) in reloc_cache_reset() argument
1066 reloc_gpu_flush(eb, cache); in reloc_cache_reset()
1140 struct i915_execbuffer *eb, in reloc_iomap() argument
1143 struct reloc_cache *cache = &eb->reloc_cache; in reloc_iomap()
1165 vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, in reloc_iomap()
1207 struct i915_execbuffer *eb, in reloc_vaddr() argument
1210 struct reloc_cache *cache = &eb->reloc_cache; in reloc_vaddr()
1218 vaddr = reloc_iomap(obj, eb, page); in reloc_vaddr()
1267 static int __reloc_gpu_alloc(struct i915_execbuffer *eb, in __reloc_gpu_alloc() argument
1272 struct reloc_cache *cache = &eb->reloc_cache; in __reloc_gpu_alloc()
1273 struct intel_gt_buffer_pool_node *pool = eb->reloc_pool; in __reloc_gpu_alloc()
1284 eb->reloc_pool = NULL; in __reloc_gpu_alloc()
1286 err = i915_gem_object_lock(pool->obj, &eb->ww); in __reloc_gpu_alloc()
1305 err = i915_vma_pin_ww(batch, &eb->ww, 0, 0, PIN_USER | PIN_NONBLOCK); in __reloc_gpu_alloc()
1309 if (engine == eb->context->engine) { in __reloc_gpu_alloc()
1310 rq = i915_request_create(eb->context); in __reloc_gpu_alloc()
1312 struct intel_context *ce = eb->reloc_context; in __reloc_gpu_alloc()
1322 ce->vm = i915_vm_get(eb->context->vm); in __reloc_gpu_alloc()
1323 eb->reloc_context = ce; in __reloc_gpu_alloc()
1326 err = intel_context_pin_ww(ce, &eb->ww); in __reloc_gpu_alloc()
1346 err = eb->engine->emit_bb_start(rq, in __reloc_gpu_alloc()
1379 eb->reloc_pool = pool; in __reloc_gpu_alloc()
1388 static u32 *reloc_gpu(struct i915_execbuffer *eb, in reloc_gpu() argument
1392 struct reloc_cache *cache = &eb->reloc_cache; in reloc_gpu()
1396 reloc_gpu_flush(eb, cache); in reloc_gpu()
1400 struct intel_engine_cs *engine = eb->engine; in reloc_gpu()
1408 err = __reloc_gpu_alloc(eb, engine, vma, len); in reloc_gpu()
1444 static int __reloc_entry_gpu(struct i915_execbuffer *eb, in __reloc_entry_gpu() argument
1449 const unsigned int gen = eb->reloc_cache.gen; in __reloc_entry_gpu()
1461 batch = reloc_gpu(eb, vma, len); in __reloc_entry_gpu()
1493 } else if (IS_I965G(eb->i915)) { in __reloc_entry_gpu()
1504 !(IS_I915G(eb->i915) || IS_I915GM(eb->i915))) { in __reloc_entry_gpu()
1517 static int reloc_entry_gpu(struct i915_execbuffer *eb, in reloc_entry_gpu() argument
1522 if (eb->reloc_cache.vaddr) in reloc_entry_gpu()
1528 return __reloc_entry_gpu(eb, vma, offset, target_addr); in reloc_entry_gpu()
1534 struct i915_execbuffer *eb, in relocate_entry() argument
1539 int reloc_gpu = reloc_entry_gpu(eb, vma, offset, target_addr); in relocate_entry()
1545 bool wide = eb->reloc_cache.use_64bit_reloc; in relocate_entry()
1549 vaddr = reloc_vaddr(vma->obj, eb, in relocate_entry()
1557 eb->reloc_cache.vaddr); in relocate_entry()
1571 eb_relocate_entry(struct i915_execbuffer *eb, in eb_relocate_entry() argument
1575 struct drm_i915_private *i915 = eb->i915; in eb_relocate_entry()
1580 target = eb_get_vma(eb, reloc->target_handle); in eb_relocate_entry()
1617 IS_GEN(eb->i915, 6)) { in eb_relocate_entry()
1636 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { in eb_relocate_entry()
1663 return relocate_entry(ev->vma, reloc, eb, target->vma); in eb_relocate_entry()
1666 static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) in eb_relocate_vma() argument
1710 u64 offset = eb_relocate_entry(eb, ev, r); in eb_relocate_vma()
1746 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma()
1751 eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev) in eb_relocate_vma_slow() argument
1760 u64 offset = eb_relocate_entry(eb, ev, &relocs[i]); in eb_relocate_vma_slow()
1769 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma_slow()
1800 static int eb_copy_relocations(const struct i915_execbuffer *eb) in eb_copy_relocations() argument
1803 const unsigned int count = eb->buffer_count; in eb_copy_relocations()
1808 const unsigned int nreloc = eb->exec[i].relocation_count; in eb_copy_relocations()
1816 err = check_relocations(&eb->exec[i]); in eb_copy_relocations()
1820 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); in eb_copy_relocations()
1862 eb->exec[i].relocs_ptr = (uintptr_t)relocs; in eb_copy_relocations()
1874 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); in eb_copy_relocations()
1875 if (eb->exec[i].relocation_count) in eb_copy_relocations()
1881 static int eb_prefault_relocations(const struct i915_execbuffer *eb) in eb_prefault_relocations() argument
1883 const unsigned int count = eb->buffer_count; in eb_prefault_relocations()
1889 err = check_relocations(&eb->exec[i]); in eb_prefault_relocations()
1897 static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb, in eb_relocate_parse_slow() argument
1911 eb_release_vmas(eb, false); in eb_relocate_parse_slow()
1912 i915_gem_ww_ctx_fini(&eb->ww); in eb_relocate_parse_slow()
1943 err = eb_prefault_relocations(eb); in eb_relocate_parse_slow()
1945 err = eb_copy_relocations(eb); in eb_relocate_parse_slow()
1953 flush_workqueue(eb->i915->mm.userptr_wq); in eb_relocate_parse_slow()
1956 i915_gem_ww_ctx_init(&eb->ww, true); in eb_relocate_parse_slow()
1962 rq = eb_pin_engine(eb, false); in eb_relocate_parse_slow()
1972 err = eb_validate_vmas(eb); in eb_relocate_parse_slow()
1976 GEM_BUG_ON(!eb->batch); in eb_relocate_parse_slow()
1978 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse_slow()
1981 err = eb_relocate_vma(eb, ev); in eb_relocate_parse_slow()
1986 err = eb_relocate_vma_slow(eb, ev); in eb_relocate_parse_slow()
2002 err = eb_parse(eb); in eb_relocate_parse_slow()
2015 eb_release_vmas(eb, false); in eb_relocate_parse_slow()
2016 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse_slow()
2026 const unsigned int count = eb->buffer_count; in eb_relocate_parse_slow()
2031 &eb->exec[i]; in eb_relocate_parse_slow()
2048 static int eb_relocate_parse(struct i915_execbuffer *eb) in eb_relocate_parse() argument
2055 rq = eb_pin_engine(eb, throttle); in eb_relocate_parse()
2066 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; in eb_relocate_parse()
2085 err = eb_validate_vmas(eb); in eb_relocate_parse()
2092 if (eb->args->flags & __EXEC_HAS_RELOC) { in eb_relocate_parse()
2095 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse()
2096 err = eb_relocate_vma(eb, ev); in eb_relocate_parse()
2108 err = eb_parse(eb); in eb_relocate_parse()
2112 eb_release_vmas(eb, false); in eb_relocate_parse()
2113 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse()
2121 err = eb_relocate_parse_slow(eb, rq); in eb_relocate_parse()
2130 eb->args->flags &= ~__EXEC_HAS_RELOC; in eb_relocate_parse()
2135 static int eb_move_to_gpu(struct i915_execbuffer *eb) in eb_move_to_gpu() argument
2137 const unsigned int count = eb->buffer_count; in eb_move_to_gpu()
2142 struct eb_vma *ev = &eb->vma[i]; in eb_move_to_gpu()
2154 capture->next = eb->request->capture_list; in eb_move_to_gpu()
2156 eb->request->capture_list = capture; in eb_move_to_gpu()
2179 (eb->request, obj, flags & EXEC_OBJECT_WRITE); in eb_move_to_gpu()
2183 err = i915_vma_move_to_active(vma, eb->request, flags); in eb_move_to_gpu()
2190 intel_gt_chipset_flush(eb->engine->gt); in eb_move_to_gpu()
2194 i915_request_set_error_once(eb->request, err); in eb_move_to_gpu()
2249 shadow_batch_pin(struct i915_execbuffer *eb, in shadow_batch_pin() argument
2261 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags); in shadow_batch_pin()
2338 static int eb_parse_pipeline(struct i915_execbuffer *eb, in eb_parse_pipeline() argument
2345 GEM_BUG_ON(overflows_type(eb->batch_start_offset, pw->batch_offset)); in eb_parse_pipeline()
2346 GEM_BUG_ON(overflows_type(eb->batch_len, pw->batch_length)); in eb_parse_pipeline()
2352 err = i915_active_acquire(&eb->batch->vma->active); in eb_parse_pipeline()
2368 pw->engine = eb->engine; in eb_parse_pipeline()
2369 pw->batch = eb->batch->vma; in eb_parse_pipeline()
2370 pw->batch_offset = eb->batch_start_offset; in eb_parse_pipeline()
2371 pw->batch_length = eb->batch_len; in eb_parse_pipeline()
2376 err = parser_mark_active(pw, eb->context->timeline); in eb_parse_pipeline()
2408 i915_active_release(&eb->batch->vma->active); in eb_parse_pipeline()
2414 static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma) in eb_dispatch_secure() argument
2420 if (eb->batch_flags & I915_DISPATCH_SECURE) in eb_dispatch_secure()
2421 return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0); in eb_dispatch_secure()
2426 static int eb_parse(struct i915_execbuffer *eb) in eb_parse() argument
2428 struct drm_i915_private *i915 = eb->i915; in eb_parse()
2429 struct intel_gt_buffer_pool_node *pool = eb->batch_pool; in eb_parse()
2434 if (!eb_use_cmdparser(eb)) { in eb_parse()
2435 batch = eb_dispatch_secure(eb, eb->batch->vma); in eb_parse()
2442 len = eb->batch_len; in eb_parse()
2443 if (!CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2448 if (!eb->context->vm->has_read_only) { in eb_parse()
2456 if (unlikely(len < eb->batch_len)) /* last paranoid check of overflow */ in eb_parse()
2460 pool = intel_gt_get_buffer_pool(eb->engine->gt, len); in eb_parse()
2463 eb->batch_pool = pool; in eb_parse()
2466 err = i915_gem_object_lock(pool->obj, &eb->ww); in eb_parse()
2470 shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER); in eb_parse()
2479 if (CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2482 shadow = shadow_batch_pin(eb, pool->obj, in eb_parse()
2483 &eb->engine->gt->ggtt->vm, in eb_parse()
2492 eb->batch_flags |= I915_DISPATCH_SECURE; in eb_parse()
2495 batch = eb_dispatch_secure(eb, shadow); in eb_parse()
2501 err = eb_parse_pipeline(eb, shadow, trampoline); in eb_parse()
2505 eb->batch = &eb->vma[eb->buffer_count++]; in eb_parse()
2506 eb->batch->vma = i915_vma_get(shadow); in eb_parse()
2507 eb->batch->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2509 eb->trampoline = trampoline; in eb_parse()
2510 eb->batch_start_offset = 0; in eb_parse()
2514 eb->batch = &eb->vma[eb->buffer_count++]; in eb_parse()
2515 eb->batch->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2516 eb->batch->vma = i915_vma_get(batch); in eb_parse()
2532 static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) in eb_submit() argument
2536 err = eb_move_to_gpu(eb); in eb_submit()
2540 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { in eb_submit()
2541 err = i915_reset_gen7_sol_offsets(eb->request); in eb_submit()
2552 if (eb->engine->emit_init_breadcrumb) { in eb_submit()
2553 err = eb->engine->emit_init_breadcrumb(eb->request); in eb_submit()
2558 err = eb->engine->emit_bb_start(eb->request, in eb_submit()
2560 eb->batch_start_offset, in eb_submit()
2561 eb->batch_len, in eb_submit()
2562 eb->batch_flags); in eb_submit()
2566 if (eb->trampoline) { in eb_submit()
2567 GEM_BUG_ON(eb->batch_start_offset); in eb_submit()
2568 err = eb->engine->emit_bb_start(eb->request, in eb_submit()
2569 eb->trampoline->node.start + in eb_submit()
2570 eb->batch_len, in eb_submit()
2576 if (intel_context_nopreempt(eb->context)) in eb_submit()
2577 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &eb->request->fence.flags); in eb_submit()
2613 static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce) in eb_throttle() argument
2647 static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, bool throttle) in eb_pin_engine() argument
2649 struct intel_context *ce = eb->context; in eb_pin_engine()
2654 GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); in eb_pin_engine()
2664 err = intel_context_pin_ww(ce, &eb->ww); in eb_pin_engine()
2684 rq = eb_throttle(eb, ce); in eb_pin_engine()
2687 eb->args->flags |= __EXEC_ENGINE_PINNED; in eb_pin_engine()
2691 static void eb_unpin_engine(struct i915_execbuffer *eb) in eb_unpin_engine() argument
2693 struct intel_context *ce = eb->context; in eb_unpin_engine()
2696 if (!(eb->args->flags & __EXEC_ENGINE_PINNED)) in eb_unpin_engine()
2699 eb->args->flags &= ~__EXEC_ENGINE_PINNED; in eb_unpin_engine()
2709 eb_select_legacy_ring(struct i915_execbuffer *eb) in eb_select_legacy_ring() argument
2711 struct drm_i915_private *i915 = eb->i915; in eb_select_legacy_ring()
2712 struct drm_i915_gem_execbuffer2 *args = eb->args; in eb_select_legacy_ring()
2727 bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file); in eb_select_legacy_ring()
2752 eb_select_engine(struct i915_execbuffer *eb) in eb_select_engine() argument
2758 if (i915_gem_context_user_engines(eb->gem_context)) in eb_select_engine()
2759 idx = eb->args->flags & I915_EXEC_RING_MASK; in eb_select_engine()
2761 idx = eb_select_legacy_ring(eb); in eb_select_engine()
2763 ce = i915_gem_context_get_engine(eb->gem_context, idx); in eb_select_engine()
2783 eb->context = ce; in eb_select_engine()
2784 eb->engine = ce->engine; in eb_select_engine()
2800 eb_put_engine(struct i915_execbuffer *eb) in eb_put_engine() argument
2802 intel_gt_pm_put(eb->engine->gt); in eb_put_engine()
2803 intel_context_put(eb->context); in eb_put_engine()
2818 add_timeline_fence_array(struct i915_execbuffer *eb, in add_timeline_fence_array() argument
2835 SIZE_MAX / sizeof(*f)) - eb->num_fences) in add_timeline_fence_array()
2846 f = krealloc(eb->fences, in add_timeline_fence_array()
2847 (eb->num_fences + nfences) * sizeof(*f), in add_timeline_fence_array()
2852 eb->fences = f; in add_timeline_fence_array()
2853 f += eb->num_fences; in add_timeline_fence_array()
2875 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_timeline_fence_array()
2942 eb->num_fences++; in add_timeline_fence_array()
2948 static int add_fence_array(struct i915_execbuffer *eb) in add_fence_array() argument
2950 struct drm_i915_gem_execbuffer2 *args = eb->args; in add_fence_array()
2965 SIZE_MAX / sizeof(*f) - eb->num_fences)) in add_fence_array()
2972 f = krealloc(eb->fences, in add_fence_array()
2973 (eb->num_fences + num_fences) * sizeof(*f), in add_fence_array()
2978 eb->fences = f; in add_fence_array()
2979 f += eb->num_fences; in add_fence_array()
2991 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_fence_array()
3014 eb->num_fences++; in add_fence_array()
3027 await_fence_array(struct i915_execbuffer *eb) in await_fence_array() argument
3032 for (n = 0; n < eb->num_fences; n++) { in await_fence_array()
3036 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); in await_fence_array()
3038 if (!eb->fences[n].dma_fence) in await_fence_array()
3041 err = i915_request_await_dma_fence(eb->request, in await_fence_array()
3042 eb->fences[n].dma_fence); in await_fence_array()
3050 static void signal_fence_array(const struct i915_execbuffer *eb) in signal_fence_array() argument
3052 struct dma_fence * const fence = &eb->request->fence; in signal_fence_array()
3055 for (n = 0; n < eb->num_fences; n++) { in signal_fence_array()
3059 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); in signal_fence_array()
3063 if (eb->fences[n].chain_fence) { in signal_fence_array()
3065 eb->fences[n].chain_fence, in signal_fence_array()
3067 eb->fences[n].value); in signal_fence_array()
3072 eb->fences[n].chain_fence = NULL; in signal_fence_array()
3082 struct i915_execbuffer *eb = data; in parse_timeline_fences() local
3088 return add_timeline_fence_array(eb, &timeline_fences); in parse_timeline_fences()
3100 static int eb_request_add(struct i915_execbuffer *eb, int err) in eb_request_add() argument
3102 struct i915_request *rq = eb->request; in eb_request_add()
3115 if (likely(!intel_context_is_closed(eb->context))) { in eb_request_add()
3116 attr = eb->gem_context->sched; in eb_request_add()
3141 struct i915_execbuffer *eb) in parse_execbuf2_extensions() argument
3149 if (eb->args->flags & I915_EXEC_FENCE_ARRAY) in parse_execbuf2_extensions()
3158 eb); in parse_execbuf2_extensions()
3168 struct i915_execbuffer eb; in i915_gem_do_execbuffer() local
3179 eb.i915 = i915; in i915_gem_do_execbuffer()
3180 eb.file = file; in i915_gem_do_execbuffer()
3181 eb.args = args; in i915_gem_do_execbuffer()
3185 eb.exec = exec; in i915_gem_do_execbuffer()
3186 eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); in i915_gem_do_execbuffer()
3187 eb.vma[0].vma = NULL; in i915_gem_do_execbuffer()
3188 eb.reloc_pool = eb.batch_pool = NULL; in i915_gem_do_execbuffer()
3189 eb.reloc_context = NULL; in i915_gem_do_execbuffer()
3191 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; in i915_gem_do_execbuffer()
3192 reloc_cache_init(&eb.reloc_cache, eb.i915); in i915_gem_do_execbuffer()
3194 eb.buffer_count = args->buffer_count; in i915_gem_do_execbuffer()
3195 eb.batch_start_offset = args->batch_start_offset; in i915_gem_do_execbuffer()
3196 eb.batch_len = args->batch_len; in i915_gem_do_execbuffer()
3197 eb.trampoline = NULL; in i915_gem_do_execbuffer()
3199 eb.fences = NULL; in i915_gem_do_execbuffer()
3200 eb.num_fences = 0; in i915_gem_do_execbuffer()
3202 eb.batch_flags = 0; in i915_gem_do_execbuffer()
3214 eb.batch_flags |= I915_DISPATCH_SECURE; in i915_gem_do_execbuffer()
3217 eb.batch_flags |= I915_DISPATCH_PINNED; in i915_gem_do_execbuffer()
3219 err = parse_execbuf2_extensions(args, &eb); in i915_gem_do_execbuffer()
3223 err = add_fence_array(&eb); in i915_gem_do_execbuffer()
3248 err = eb_create(&eb); in i915_gem_do_execbuffer()
3252 GEM_BUG_ON(!eb.lut_size); in i915_gem_do_execbuffer()
3254 err = eb_select_context(&eb); in i915_gem_do_execbuffer()
3258 err = eb_select_engine(&eb); in i915_gem_do_execbuffer()
3262 err = eb_lookup_vmas(&eb); in i915_gem_do_execbuffer()
3264 eb_release_vmas(&eb, true); in i915_gem_do_execbuffer()
3268 i915_gem_ww_ctx_init(&eb.ww, true); in i915_gem_do_execbuffer()
3270 err = eb_relocate_parse(&eb); in i915_gem_do_execbuffer()
3283 ww_acquire_done(&eb.ww.ctx); in i915_gem_do_execbuffer()
3285 batch = eb.batch->vma; in i915_gem_do_execbuffer()
3288 GEM_BUG_ON(eb.reloc_cache.rq); in i915_gem_do_execbuffer()
3291 eb.request = i915_request_create(eb.context); in i915_gem_do_execbuffer()
3292 if (IS_ERR(eb.request)) { in i915_gem_do_execbuffer()
3293 err = PTR_ERR(eb.request); in i915_gem_do_execbuffer()
3299 err = i915_request_await_execution(eb.request, in i915_gem_do_execbuffer()
3301 eb.engine->bond_execute); in i915_gem_do_execbuffer()
3303 err = i915_request_await_dma_fence(eb.request, in i915_gem_do_execbuffer()
3309 if (eb.fences) { in i915_gem_do_execbuffer()
3310 err = await_fence_array(&eb); in i915_gem_do_execbuffer()
3316 out_fence = sync_file_create(&eb.request->fence); in i915_gem_do_execbuffer()
3330 eb.request->batch = batch; in i915_gem_do_execbuffer()
3331 if (eb.batch_pool) in i915_gem_do_execbuffer()
3332 intel_gt_buffer_pool_mark_active(eb.batch_pool, eb.request); in i915_gem_do_execbuffer()
3334 trace_i915_request_queue(eb.request, eb.batch_flags); in i915_gem_do_execbuffer()
3335 err = eb_submit(&eb, batch); in i915_gem_do_execbuffer()
3337 i915_request_get(eb.request); in i915_gem_do_execbuffer()
3338 err = eb_request_add(&eb, err); in i915_gem_do_execbuffer()
3340 if (eb.fences) in i915_gem_do_execbuffer()
3341 signal_fence_array(&eb); in i915_gem_do_execbuffer()
3353 i915_request_put(eb.request); in i915_gem_do_execbuffer()
3356 eb_release_vmas(&eb, true); in i915_gem_do_execbuffer()
3357 if (eb.trampoline) in i915_gem_do_execbuffer()
3358 i915_vma_unpin(eb.trampoline); in i915_gem_do_execbuffer()
3360 i915_gem_ww_ctx_fini(&eb.ww); in i915_gem_do_execbuffer()
3362 if (eb.batch_pool) in i915_gem_do_execbuffer()
3363 intel_gt_buffer_pool_put(eb.batch_pool); in i915_gem_do_execbuffer()
3364 if (eb.reloc_pool) in i915_gem_do_execbuffer()
3365 intel_gt_buffer_pool_put(eb.reloc_pool); in i915_gem_do_execbuffer()
3366 if (eb.reloc_context) in i915_gem_do_execbuffer()
3367 intel_context_put(eb.reloc_context); in i915_gem_do_execbuffer()
3369 eb_put_engine(&eb); in i915_gem_do_execbuffer()
3371 i915_gem_context_put(eb.gem_context); in i915_gem_do_execbuffer()
3373 eb_destroy(&eb); in i915_gem_do_execbuffer()
3380 put_fence_array(eb.fences, eb.num_fences); in i915_gem_do_execbuffer()