Lines Matching refs:kfd
501 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
503 static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
505 static int kfd_resume(struct kfd_dev *kfd);
527 struct kfd_dev *kfd; in kgd2kfd_probe() local
536 kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); in kgd2kfd_probe()
537 if (!kfd) in kgd2kfd_probe()
544 kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd); in kgd2kfd_probe()
546 !kfd->pci_atomic_requested) { in kgd2kfd_probe()
550 kfree(kfd); in kgd2kfd_probe()
554 kfd->kgd = kgd; in kgd2kfd_probe()
555 kfd->device_info = device_info; in kgd2kfd_probe()
556 kfd->pdev = pdev; in kgd2kfd_probe()
557 kfd->init_complete = false; in kgd2kfd_probe()
558 kfd->kfd2kgd = f2g; in kgd2kfd_probe()
559 atomic_set(&kfd->compute_profile, 0); in kgd2kfd_probe()
561 mutex_init(&kfd->doorbell_mutex); in kgd2kfd_probe()
562 memset(&kfd->doorbell_available_index, 0, in kgd2kfd_probe()
563 sizeof(kfd->doorbell_available_index)); in kgd2kfd_probe()
565 atomic_set(&kfd->sram_ecc_flag, 0); in kgd2kfd_probe()
567 return kfd; in kgd2kfd_probe()
570 static void kfd_cwsr_init(struct kfd_dev *kfd) in kfd_cwsr_init() argument
572 if (cwsr_enable && kfd->device_info->supports_cwsr) { in kfd_cwsr_init()
573 if (kfd->device_info->asic_family < CHIP_VEGA10) { in kfd_cwsr_init()
575 kfd->cwsr_isa = cwsr_trap_gfx8_hex; in kfd_cwsr_init()
576 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex); in kfd_cwsr_init()
577 } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) { in kfd_cwsr_init()
579 kfd->cwsr_isa = cwsr_trap_arcturus_hex; in kfd_cwsr_init()
580 kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex); in kfd_cwsr_init()
581 } else if (kfd->device_info->asic_family < CHIP_NAVI10) { in kfd_cwsr_init()
583 kfd->cwsr_isa = cwsr_trap_gfx9_hex; in kfd_cwsr_init()
584 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex); in kfd_cwsr_init()
587 kfd->cwsr_isa = cwsr_trap_gfx10_hex; in kfd_cwsr_init()
588 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex); in kfd_cwsr_init()
591 kfd->cwsr_enabled = true; in kfd_cwsr_init()
595 bool kgd2kfd_device_init(struct kfd_dev *kfd, in kgd2kfd_device_init() argument
600 kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, in kgd2kfd_device_init()
602 kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, in kgd2kfd_device_init()
604 kfd->shared_resources = *gpu_resources; in kgd2kfd_device_init()
606 kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1; in kgd2kfd_device_init()
607 kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1; in kgd2kfd_device_init()
608 kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd in kgd2kfd_device_init()
609 - kfd->vm_info.first_vmid_kfd + 1; in kgd2kfd_device_init()
613 || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) { in kgd2kfd_device_init()
616 hws_max_conc_proc, kfd->vm_info.vmid_num_kfd, in kgd2kfd_device_init()
617 kfd->vm_info.vmid_num_kfd); in kgd2kfd_device_init()
618 kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd; in kgd2kfd_device_init()
620 kfd->max_proc_per_quantum = hws_max_conc_proc; in kgd2kfd_device_init()
623 if (hws_gws_support && amdgpu_amdkfd_alloc_gws(kfd->kgd, in kgd2kfd_device_init()
624 amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws)) { in kgd2kfd_device_init()
626 amdgpu_amdkfd_get_num_gws(kfd->kgd)); in kgd2kfd_device_init()
631 kfd->device_info->mqd_size_aligned; in kgd2kfd_device_init()
648 kfd->kgd, size, &kfd->gtt_mem, in kgd2kfd_device_init()
649 &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, in kgd2kfd_device_init()
658 if (kfd_gtt_sa_init(kfd, size, 512) != 0) { in kgd2kfd_device_init()
663 if (kfd_doorbell_init(kfd)) { in kgd2kfd_device_init()
669 if (kfd->kfd2kgd->get_hive_id) in kgd2kfd_device_init()
670 kfd->hive_id = kfd->kfd2kgd->get_hive_id(kfd->kgd); in kgd2kfd_device_init()
672 if (kfd_interrupt_init(kfd)) { in kgd2kfd_device_init()
677 kfd->dqm = device_queue_manager_init(kfd); in kgd2kfd_device_init()
678 if (!kfd->dqm) { in kgd2kfd_device_init()
683 if (kfd_iommu_device_init(kfd)) { in kgd2kfd_device_init()
688 kfd_cwsr_init(kfd); in kgd2kfd_device_init()
690 if (kfd_resume(kfd)) in kgd2kfd_device_init()
693 kfd->dbgmgr = NULL; in kgd2kfd_device_init()
695 if (kfd_topology_add_device(kfd)) { in kgd2kfd_device_init()
700 kfd->init_complete = true; in kgd2kfd_device_init()
701 dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor, in kgd2kfd_device_init()
702 kfd->pdev->device); in kgd2kfd_device_init()
705 kfd->dqm->sched_policy); in kgd2kfd_device_init()
712 device_queue_manager_uninit(kfd->dqm); in kgd2kfd_device_init()
714 kfd_interrupt_exit(kfd); in kgd2kfd_device_init()
716 kfd_doorbell_fini(kfd); in kgd2kfd_device_init()
718 kfd_gtt_sa_fini(kfd); in kgd2kfd_device_init()
720 amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); in kgd2kfd_device_init()
723 amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); in kgd2kfd_device_init()
726 kfd->pdev->vendor, kfd->pdev->device); in kgd2kfd_device_init()
728 return kfd->init_complete; in kgd2kfd_device_init()
731 void kgd2kfd_device_exit(struct kfd_dev *kfd) in kgd2kfd_device_exit() argument
733 if (kfd->init_complete) { in kgd2kfd_device_exit()
734 kgd2kfd_suspend(kfd); in kgd2kfd_device_exit()
735 device_queue_manager_uninit(kfd->dqm); in kgd2kfd_device_exit()
736 kfd_interrupt_exit(kfd); in kgd2kfd_device_exit()
737 kfd_topology_remove_device(kfd); in kgd2kfd_device_exit()
738 kfd_doorbell_fini(kfd); in kgd2kfd_device_exit()
739 kfd_gtt_sa_fini(kfd); in kgd2kfd_device_exit()
740 amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); in kgd2kfd_device_exit()
742 amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); in kgd2kfd_device_exit()
745 kfree(kfd); in kgd2kfd_device_exit()
748 int kgd2kfd_pre_reset(struct kfd_dev *kfd) in kgd2kfd_pre_reset() argument
750 if (!kfd->init_complete) in kgd2kfd_pre_reset()
752 kgd2kfd_suspend(kfd); in kgd2kfd_pre_reset()
755 dqm_lock(kfd->dqm); in kgd2kfd_pre_reset()
757 kfd_signal_reset_event(kfd); in kgd2kfd_pre_reset()
767 int kgd2kfd_post_reset(struct kfd_dev *kfd) in kgd2kfd_post_reset() argument
771 if (!kfd->init_complete) in kgd2kfd_post_reset()
774 dqm_unlock(kfd->dqm); in kgd2kfd_post_reset()
776 ret = kfd_resume(kfd); in kgd2kfd_post_reset()
781 atomic_set(&kfd->sram_ecc_flag, 0); in kgd2kfd_post_reset()
791 void kgd2kfd_suspend(struct kfd_dev *kfd) in kgd2kfd_suspend() argument
793 if (!kfd->init_complete) in kgd2kfd_suspend()
800 kfd->dqm->ops.stop(kfd->dqm); in kgd2kfd_suspend()
802 kfd_iommu_suspend(kfd); in kgd2kfd_suspend()
805 int kgd2kfd_resume(struct kfd_dev *kfd) in kgd2kfd_resume() argument
809 if (!kfd->init_complete) in kgd2kfd_resume()
812 ret = kfd_resume(kfd); in kgd2kfd_resume()
824 static int kfd_resume(struct kfd_dev *kfd) in kfd_resume() argument
828 err = kfd_iommu_resume(kfd); in kfd_resume()
832 kfd->pdev->vendor, kfd->pdev->device); in kfd_resume()
836 err = kfd->dqm->ops.start(kfd->dqm); in kfd_resume()
840 kfd->pdev->vendor, kfd->pdev->device); in kfd_resume()
847 kfd_iommu_suspend(kfd); in kfd_resume()
852 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) in kgd2kfd_interrupt() argument
858 if (!kfd->init_complete) in kgd2kfd_interrupt()
861 if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) { in kgd2kfd_interrupt()
866 spin_lock_irqsave(&kfd->interrupt_lock, flags); in kgd2kfd_interrupt()
868 if (kfd->interrupts_active in kgd2kfd_interrupt()
869 && interrupt_is_wanted(kfd, ih_ring_entry, in kgd2kfd_interrupt()
871 && enqueue_ih_ring_entry(kfd, in kgd2kfd_interrupt()
873 queue_work(kfd->ih_wq, &kfd->interrupt_work); in kgd2kfd_interrupt()
875 spin_unlock_irqrestore(&kfd->interrupt_lock, flags); in kgd2kfd_interrupt()
964 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, in kfd_gtt_sa_init() argument
976 kfd->gtt_sa_chunk_size = chunk_size; in kfd_gtt_sa_init()
977 kfd->gtt_sa_num_of_chunks = buf_size / chunk_size; in kfd_gtt_sa_init()
979 num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) / in kfd_gtt_sa_init()
982 kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL); in kfd_gtt_sa_init()
984 if (!kfd->gtt_sa_bitmap) in kfd_gtt_sa_init()
988 kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap); in kfd_gtt_sa_init()
990 mutex_init(&kfd->gtt_sa_lock); in kfd_gtt_sa_init()
996 static void kfd_gtt_sa_fini(struct kfd_dev *kfd) in kfd_gtt_sa_fini() argument
998 mutex_destroy(&kfd->gtt_sa_lock); in kfd_gtt_sa_fini()
999 kfree(kfd->gtt_sa_bitmap); in kfd_gtt_sa_fini()
1016 int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, in kfd_gtt_sa_allocate() argument
1024 if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size) in kfd_gtt_sa_allocate()
1035 mutex_lock(&kfd->gtt_sa_lock); in kfd_gtt_sa_allocate()
1039 found = find_next_zero_bit(kfd->gtt_sa_bitmap, in kfd_gtt_sa_allocate()
1040 kfd->gtt_sa_num_of_chunks, in kfd_gtt_sa_allocate()
1046 if (found == kfd->gtt_sa_num_of_chunks) in kfd_gtt_sa_allocate()
1053 kfd->gtt_start_gpu_addr, in kfd_gtt_sa_allocate()
1055 kfd->gtt_sa_chunk_size); in kfd_gtt_sa_allocate()
1057 kfd->gtt_start_cpu_ptr, in kfd_gtt_sa_allocate()
1059 kfd->gtt_sa_chunk_size); in kfd_gtt_sa_allocate()
1065 if (size <= kfd->gtt_sa_chunk_size) { in kfd_gtt_sa_allocate()
1067 set_bit(found, kfd->gtt_sa_bitmap); in kfd_gtt_sa_allocate()
1072 cur_size = size - kfd->gtt_sa_chunk_size; in kfd_gtt_sa_allocate()
1075 find_next_zero_bit(kfd->gtt_sa_bitmap, in kfd_gtt_sa_allocate()
1076 kfd->gtt_sa_num_of_chunks, ++found); in kfd_gtt_sa_allocate()
1090 if (found == kfd->gtt_sa_num_of_chunks) in kfd_gtt_sa_allocate()
1094 if (cur_size <= kfd->gtt_sa_chunk_size) in kfd_gtt_sa_allocate()
1097 cur_size -= kfd->gtt_sa_chunk_size; in kfd_gtt_sa_allocate()
1108 set_bit(found, kfd->gtt_sa_bitmap); in kfd_gtt_sa_allocate()
1111 mutex_unlock(&kfd->gtt_sa_lock); in kfd_gtt_sa_allocate()
1116 mutex_unlock(&kfd->gtt_sa_lock); in kfd_gtt_sa_allocate()
1121 int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj) in kfd_gtt_sa_free() argument
1132 mutex_lock(&kfd->gtt_sa_lock); in kfd_gtt_sa_free()
1138 clear_bit(bit, kfd->gtt_sa_bitmap); in kfd_gtt_sa_free()
1140 mutex_unlock(&kfd->gtt_sa_lock); in kfd_gtt_sa_free()
1146 void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) in kgd2kfd_set_sram_ecc_flag() argument
1148 if (kfd) in kgd2kfd_set_sram_ecc_flag()
1149 atomic_inc(&kfd->sram_ecc_flag); in kgd2kfd_set_sram_ecc_flag()
1152 void kfd_inc_compute_active(struct kfd_dev *kfd) in kfd_inc_compute_active() argument
1154 if (atomic_inc_return(&kfd->compute_profile) == 1) in kfd_inc_compute_active()
1155 amdgpu_amdkfd_set_compute_idle(kfd->kgd, false); in kfd_inc_compute_active()
1158 void kfd_dec_compute_active(struct kfd_dev *kfd) in kfd_dec_compute_active() argument
1160 int count = atomic_dec_return(&kfd->compute_profile); in kfd_dec_compute_active()
1163 amdgpu_amdkfd_set_compute_idle(kfd->kgd, true); in kfd_dec_compute_active()