Lines Matching +full:rpm +full:- +full:requests
1 // SPDX-License-Identifier: MIT
7 #include <drm/intel-gtt.h>
41 spin_lock_init(gt->irq_lock); in intel_gt_common_init_early()
43 INIT_LIST_HEAD(>->lmem_userfault_list); in intel_gt_common_init_early()
44 mutex_init(>->lmem_userfault_lock); in intel_gt_common_init_early()
45 INIT_LIST_HEAD(>->closed_vma); in intel_gt_common_init_early()
46 spin_lock_init(>->closed_lock); in intel_gt_common_init_early()
48 init_llist_head(>->watchdog.list); in intel_gt_common_init_early()
49 INIT_WORK(>->watchdog.work, intel_gt_watchdog_work); in intel_gt_common_init_early()
55 mutex_init(>->tlb.invalidate_lock); in intel_gt_common_init_early()
56 seqcount_mutex_init(>->tlb.seqno, >->tlb.invalidate_lock); in intel_gt_common_init_early()
59 intel_uc_init_early(>->uc); in intel_gt_common_init_early()
60 intel_rps_init_early(>->rps); in intel_gt_common_init_early()
68 gt->i915 = i915; in intel_root_gt_init_early()
69 gt->uncore = &i915->uncore; in intel_root_gt_init_early()
70 gt->irq_lock = drmm_kzalloc(&i915->drm, sizeof(*gt->irq_lock), GFP_KERNEL); in intel_root_gt_init_early()
71 if (!gt->irq_lock) in intel_root_gt_init_early()
72 return -ENOMEM; in intel_root_gt_init_early()
81 struct drm_i915_private *i915 = gt->i915; in intel_gt_probe_lmem()
82 unsigned int instance = gt->info.id; in intel_gt_probe_lmem()
90 if (err == -ENODEV) in intel_gt_probe_lmem()
93 drm_err(&i915->drm, in intel_gt_probe_lmem()
99 mem->id = id; in intel_gt_probe_lmem()
100 mem->instance = instance; in intel_gt_probe_lmem()
102 intel_memory_region_set_name(mem, "local%u", mem->instance); in intel_gt_probe_lmem()
105 GEM_BUG_ON(i915->mm.regions[id]); in intel_gt_probe_lmem()
106 i915->mm.regions[id] = mem; in intel_gt_probe_lmem()
113 gt->ggtt = drmm_kzalloc(>->i915->drm, sizeof(*gt->ggtt), GFP_KERNEL); in intel_gt_assign_ggtt()
115 return gt->ggtt ? 0 : -ENOMEM; in intel_gt_assign_ggtt()
122 intel_uc_init_mmio(>->uc); in intel_gt_init_mmio()
131 struct intel_uncore *uncore = gt->uncore; in init_unused_ring()
141 struct drm_i915_private *i915 = gt->i915; in init_unused_rings()
160 struct drm_i915_private *i915 = gt->i915; in intel_gt_init_hw()
161 struct intel_uncore *uncore = gt->uncore; in intel_gt_init_hw()
164 gt->last_init_time = ktime_get(); in intel_gt_init_hw()
200 ret = intel_uc_init_hw(>->uc); in intel_gt_init_hw()
238 struct drm_i915_private *i915 = gt->i915; in intel_gt_clear_error_registers()
239 struct intel_uncore *uncore = gt->uncore; in intel_gt_clear_error_registers()
287 drm_dbg(&engine->i915->drm, "Unexpected fault\n" in gen6_check_faults()
303 struct intel_uncore *uncore = gt->uncore; in gen8_check_faults()
307 if (GRAPHICS_VER(gt->i915) >= 12) { in gen8_check_faults()
328 drm_dbg(&uncore->i915->drm, "Unexpected fault\n" in gen8_check_faults()
344 struct drm_i915_private *i915 = gt->i915; in intel_gt_check_and_clear_faults()
359 struct intel_uncore *uncore = gt->uncore; in intel_gt_flush_ggtt_writes()
373 * An uncached read (i.e. mmio) seems to be ideal for the round-trip in intel_gt_flush_ggtt_writes()
383 if (INTEL_INFO(gt->i915)->has_coherent_ggtt) in intel_gt_flush_ggtt_writes()
388 with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) { in intel_gt_flush_ggtt_writes()
391 spin_lock_irqsave(&uncore->lock, flags); in intel_gt_flush_ggtt_writes()
394 spin_unlock_irqrestore(&uncore->lock, flags); in intel_gt_flush_ggtt_writes()
401 if (GRAPHICS_VER(gt->i915) < 6) in intel_gt_chipset_flush()
407 intel_gsc_init(>->gsc, gt->i915); in intel_gt_driver_register()
409 intel_rps_driver_register(>->rps); in intel_gt_driver_register()
417 struct drm_i915_private *i915 = gt->i915; in intel_gt_init_scratch()
430 drm_err(&i915->drm, "Failed to allocate scratch page\n"); in intel_gt_init_scratch()
434 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); in intel_gt_init_scratch()
444 gt->scratch = i915_vma_make_unshrinkable(vma); in intel_gt_init_scratch()
455 i915_vma_unpin_and_release(>->scratch, 0); in intel_gt_fini_scratch()
460 if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING) in kernel_vm()
461 return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm; in kernel_vm()
463 return i915_vm_get(>->ggtt->vm); in kernel_vm()
468 struct i915_request *requests[I915_NUM_ENGINES] = {}; in __engines_record_defaults() local
476 * We load a context onto the hw (with restore-inhibit), then switch in __engines_record_defaults()
488 GEM_BUG_ON(!engine->kernel_context); in __engines_record_defaults()
515 requests[id] = i915_request_get(rq); in __engines_record_defaults()
527 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { in __engines_record_defaults()
528 err = -EIO; in __engines_record_defaults()
532 for (id = 0; id < ARRAY_SIZE(requests); id++) { in __engines_record_defaults()
536 rq = requests[id]; in __engines_record_defaults()
540 if (rq->fence.error) { in __engines_record_defaults()
541 err = -EIO; in __engines_record_defaults()
545 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags)); in __engines_record_defaults()
546 if (!rq->context->state) in __engines_record_defaults()
550 state = shmem_create_from_object(rq->context->state->obj); in __engines_record_defaults()
555 rq->engine->default_state = state; in __engines_record_defaults()
561 * and ready to be torn-down. The quickest way we can accomplish in __engines_record_defaults()
567 for (id = 0; id < ARRAY_SIZE(requests); id++) { in __engines_record_defaults()
571 rq = requests[id]; in __engines_record_defaults()
575 ce = rq->context; in __engines_record_defaults()
593 err = -EIO; in __engines_verify_workarounds()
597 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) in __engines_verify_workarounds()
598 err = -EIO; in __engines_verify_workarounds()
617 /* If the device is asleep, we have no requests outstanding */ in intel_gt_wait_for_idle()
625 return -EINTR; in intel_gt_wait_for_idle()
634 return intel_uc_wait_for_idle(>->uc, remaining_timeout); in intel_gt_wait_for_idle()
641 err = i915_inject_probe_error(gt->i915, -ENODEV); in intel_gt_init()
654 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); in intel_gt_init()
657 GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K); in intel_gt_init()
663 gt->vm = kernel_vm(gt); in intel_gt_init()
664 if (!gt->vm) { in intel_gt_init()
665 err = -ENOMEM; in intel_gt_init()
675 err = intel_uc_init(>->uc); in intel_gt_init()
685 drm_err(>->i915->drm, "Failed to retrieve hwconfig table: %pe\n", in intel_gt_init()
696 intel_uc_init_late(>->uc); in intel_gt_init()
698 err = i915_inject_probe_error(gt->i915, -EIO); in intel_gt_init()
702 intel_migrate_init(>->migrate, gt); in intel_gt_init()
704 intel_pxp_init(>->pxp); in intel_gt_init()
709 intel_uc_fini_hw(>->uc); in intel_gt_init()
711 intel_uc_fini(>->uc); in intel_gt_init()
714 i915_vm_put(fetch_and_zero(>->vm)); in intel_gt_init()
721 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); in intel_gt_init()
729 intel_migrate_fini(>->migrate); in intel_gt_driver_remove()
730 intel_uc_driver_remove(>->uc); in intel_gt_driver_remove()
742 intel_rps_driver_unregister(>->rps); in intel_gt_driver_unregister()
743 intel_gsc_fini(>->gsc); in intel_gt_driver_unregister()
745 intel_pxp_fini(>->pxp); in intel_gt_driver_unregister()
749 * all in-flight requests so that we can quickly unbind the active in intel_gt_driver_unregister()
755 with_intel_runtime_pm(gt->uncore->rpm, wakeref) in intel_gt_driver_unregister()
763 vm = fetch_and_zero(>->vm); in intel_gt_driver_release()
767 intel_wa_list_free(>->wa_list); in intel_gt_driver_release()
783 intel_uc_driver_late_release(>->uc); in intel_gt_driver_late_release_all()
787 mutex_destroy(>->tlb.invalidate_lock); in intel_gt_driver_late_release_all()
800 uncore = drmm_kzalloc(>->i915->drm, sizeof(*uncore), GFP_KERNEL); in intel_gt_tile_setup()
802 return -ENOMEM; in intel_gt_tile_setup()
804 irq_lock = drmm_kzalloc(>->i915->drm, sizeof(*irq_lock), GFP_KERNEL); in intel_gt_tile_setup()
806 return -ENOMEM; in intel_gt_tile_setup()
808 gt->uncore = uncore; in intel_gt_tile_setup()
809 gt->irq_lock = irq_lock; in intel_gt_tile_setup()
814 intel_uncore_init_early(gt->uncore, gt); in intel_gt_tile_setup()
815 intel_wakeref_auto_init(>->userfault_wakeref, gt->uncore->rpm); in intel_gt_tile_setup()
817 ret = intel_uncore_setup_mmio(gt->uncore, phys_addr); in intel_gt_tile_setup()
821 gt->phys_addr = phys_addr; in intel_gt_tile_setup()
828 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); in intel_gt_probe_all()
829 struct intel_gt *gt = &i915->gt0; in intel_gt_probe_all()
844 gt->i915 = i915; in intel_gt_probe_all()
845 gt->name = "Primary GT"; in intel_gt_probe_all()
846 gt->info.engine_mask = RUNTIME_INFO(i915)->platform_engine_mask; in intel_gt_probe_all()
848 drm_dbg(&i915->drm, "Setting up %s\n", gt->name); in intel_gt_probe_all()
853 i915->gt[0] = gt; in intel_gt_probe_all()
858 for (i = 1, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1]; in intel_gt_probe_all()
859 gtdef->name != NULL; in intel_gt_probe_all()
860 i++, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1]) { in intel_gt_probe_all()
861 gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL); in intel_gt_probe_all()
863 ret = -ENOMEM; in intel_gt_probe_all()
867 gt->i915 = i915; in intel_gt_probe_all()
868 gt->name = gtdef->name; in intel_gt_probe_all()
869 gt->type = gtdef->type; in intel_gt_probe_all()
870 gt->info.engine_mask = gtdef->engine_mask; in intel_gt_probe_all()
871 gt->info.id = i; in intel_gt_probe_all()
873 drm_dbg(&i915->drm, "Setting up %s\n", gt->name); in intel_gt_probe_all()
875 gtdef->mapping_base, in intel_gt_probe_all()
878 ret = -ENODEV; in intel_gt_probe_all()
882 switch (gtdef->type) { in intel_gt_probe_all()
884 ret = intel_gt_tile_setup(gt, phys_addr + gtdef->mapping_base); in intel_gt_probe_all()
888 ret = intel_sa_mediagt_setup(gt, phys_addr + gtdef->mapping_base, in intel_gt_probe_all()
889 gtdef->gsi_offset); in intel_gt_probe_all()
895 MISSING_CASE(gtdef->type); in intel_gt_probe_all()
896 ret = -ENODEV; in intel_gt_probe_all()
902 i915->gt[i] = gt; in intel_gt_probe_all()
908 i915_probe_error(i915, "Failed to initialize %s! (%d)\n", gtdef->name, ret); in intel_gt_probe_all()
935 i915->gt[id] = NULL; in intel_gt_release_all()
941 drm_printf(p, "available engines: %x\n", info->engine_mask); in intel_gt_info_print()
943 intel_sseu_dump(&info->sseu, p); in intel_gt_info_print()
955 const unsigned int class = engine->class; in get_reg_and_bit()
958 if (drm_WARN_ON_ONCE(&engine->i915->drm, in get_reg_and_bit()
964 rb.reg.reg += 4 * engine->instance; /* GEN8_M2TCR */ in get_reg_and_bit()
966 rb.bit = engine->instance; in get_reg_and_bit()
988 struct drm_i915_private *i915 = gt->i915; in mmio_invalidate_full()
989 struct intel_uncore *uncore = gt->uncore; in mmio_invalidate_full()
1006 if (drm_WARN_ONCE(&i915->drm, !num, in mmio_invalidate_full()
1012 spin_lock_irq(&uncore->lock); /* serialise invalidate with GT reset */ in mmio_invalidate_full()
1025 if (GRAPHICS_VER(i915) == 12 && (engine->class == VIDEO_DECODE_CLASS || in mmio_invalidate_full()
1026 engine->class == VIDEO_ENHANCEMENT_CLASS || in mmio_invalidate_full()
1027 engine->class == COMPUTE_CLASS)) in mmio_invalidate_full()
1031 awake |= engine->mask; in mmio_invalidate_full()
1036 /* Wa_2207587034:tgl,dg1,rkl,adl-s,adl-p */ in mmio_invalidate_full()
1045 spin_unlock_irq(&uncore->lock); in mmio_invalidate_full()
1063 drm_err_ratelimited(>->i915->drm, in mmio_invalidate_full()
1065 engine->name, timeout_ms); in mmio_invalidate_full()
1082 return (s32)(cur - ALIGN(seqno, 2)) > 0; in tlb_seqno_passed()
1089 if (I915_SELFTEST_ONLY(gt->awake == -ENODEV)) in intel_gt_invalidate_tlb()
1099 mutex_lock(>->tlb.invalidate_lock); in intel_gt_invalidate_tlb()
1105 write_seqcount_invalidate(>->tlb.seqno); in intel_gt_invalidate_tlb()
1107 mutex_unlock(>->tlb.invalidate_lock); in intel_gt_invalidate_tlb()