Lines Matching +full:proc +full:- +full:id
30 #include <linux/amd-iommu.h>
59 /* Ordered, single-threaded workqueue for restoring evicted
62 * their BOs and result in a live-lock situation where processes
113 pdd = workarea->pdd; in kfd_sdma_activity_worker()
116 dqm = pdd->dev->dqm; in kfd_sdma_activity_worker()
117 qpd = &pdd->qpd; in kfd_sdma_activity_worker()
124 * we loop over all SDMA queues and get their counts from user-space. in kfd_sdma_activity_worker()
130 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list, in kfd_sdma_activity_worker()
136 * from the qpd->queues_list. in kfd_sdma_activity_worker()
137 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted. in kfd_sdma_activity_worker()
149 list_for_each_entry(q, &qpd->queues_list, list) { in kfd_sdma_activity_worker()
150 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && in kfd_sdma_activity_worker()
151 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) in kfd_sdma_activity_worker()
160 INIT_LIST_HEAD(&sdma_q->list); in kfd_sdma_activity_worker()
161 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr; in kfd_sdma_activity_worker()
162 sdma_q->queue_id = q->properties.queue_id; in kfd_sdma_activity_worker()
163 list_add_tail(&sdma_q->list, &sdma_q_list.list); in kfd_sdma_activity_worker()
168 * qpd->queues_list. Return the past activity count as the total sdma in kfd_sdma_activity_worker()
172 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
182 mm = get_task_mm(pdd->process->lead_thread); in kfd_sdma_activity_worker()
190 ret = read_sdma_queue_counter(sdma_q->rptr, &val); in kfd_sdma_activity_worker()
192 pr_debug("Failed to read SDMA queue active counter for queue id: %d", in kfd_sdma_activity_worker()
193 sdma_q->queue_id); in kfd_sdma_activity_worker()
195 sdma_q->sdma_val = val; in kfd_sdma_activity_worker()
196 workarea->sdma_activity_counter += val; in kfd_sdma_activity_worker()
209 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
211 list_for_each_entry(q, &qpd->queues_list, list) { in kfd_sdma_activity_worker()
215 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && in kfd_sdma_activity_worker()
216 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) in kfd_sdma_activity_worker()
220 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) && in kfd_sdma_activity_worker()
221 (sdma_q->queue_id == q->properties.queue_id)) { in kfd_sdma_activity_worker()
222 list_del(&sdma_q->list); in kfd_sdma_activity_worker()
233 * from qpd->queues_list during SDMA usage read. Subtract the SDMA in kfd_sdma_activity_worker()
237 workarea->sdma_activity_counter -= sdma_q->sdma_val; in kfd_sdma_activity_worker()
238 list_del(&sdma_q->list); in kfd_sdma_activity_worker()
246 list_del(&sdma_q->list); in kfd_sdma_activity_worker()
252 * @kfd_get_cu_occupancy - Collect number of waves in-flight on this device
270 struct kfd_process *proc = NULL; in kfd_get_cu_occupancy() local
274 dev = pdd->dev; in kfd_get_cu_occupancy()
275 if (dev->kfd2kgd->get_cu_occupancy == NULL) in kfd_get_cu_occupancy()
276 return -EINVAL; in kfd_get_cu_occupancy()
279 proc = pdd->process; in kfd_get_cu_occupancy()
280 if (pdd->qpd.queue_count == 0) { in kfd_get_cu_occupancy()
281 pr_debug("Gpu-Id: %d has no active queues for process %d\n", in kfd_get_cu_occupancy()
282 dev->id, proc->pasid); in kfd_get_cu_occupancy()
289 dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt, in kfd_get_cu_occupancy()
293 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu; in kfd_get_cu_occupancy()
300 if (strcmp(attr->name, "pasid") == 0) { in kfd_procfs_show()
304 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid); in kfd_procfs_show()
305 } else if (strncmp(attr->name, "vram_", 5) == 0) { in kfd_procfs_show()
308 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage)); in kfd_procfs_show()
309 } else if (strncmp(attr->name, "sdma_", 5) == 0) { in kfd_procfs_show()
329 return -EINVAL; in kfd_procfs_show()
358 &kfd_device->kobj, "proc"); in kfd_procfs_init()
360 pr_warn("Could not create procfs proc folder"); in kfd_procfs_init()
380 if (!strcmp(attr->name, "size")) in kfd_procfs_queue_show()
382 q->properties.queue_size); in kfd_procfs_queue_show()
383 else if (!strcmp(attr->name, "type")) in kfd_procfs_queue_show()
384 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type); in kfd_procfs_queue_show()
385 else if (!strcmp(attr->name, "gpuid")) in kfd_procfs_queue_show()
386 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id); in kfd_procfs_queue_show()
396 if (strcmp(attr->name, "evicted_ms") == 0) { in kfd_procfs_stats_show()
402 evict_jiffies = atomic64_read(&pdd->evict_duration_counter); in kfd_procfs_stats_show()
410 } else if (strcmp(attr->name, "cu_occupancy") == 0) { in kfd_procfs_stats_show()
424 if (!strcmp(attr->name, "faults")) { in kfd_sysfs_counters_show()
427 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); in kfd_sysfs_counters_show()
429 if (!strcmp(attr->name, "page_in")) { in kfd_sysfs_counters_show()
432 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); in kfd_sysfs_counters_show()
434 if (!strcmp(attr->name, "page_out")) { in kfd_sysfs_counters_show()
437 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); in kfd_sysfs_counters_show()
493 struct kfd_process *proc; in kfd_procfs_add_queue() local
496 if (!q || !q->process) in kfd_procfs_add_queue()
497 return -EINVAL; in kfd_procfs_add_queue()
498 proc = q->process; in kfd_procfs_add_queue()
500 /* Create proc/<pid>/queues/<queue id> folder */ in kfd_procfs_add_queue()
501 if (!proc->kobj_queues) in kfd_procfs_add_queue()
502 return -EFAULT; in kfd_procfs_add_queue()
503 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type, in kfd_procfs_add_queue()
504 proc->kobj_queues, "%u", q->properties.queue_id); in kfd_procfs_add_queue()
506 pr_warn("Creating proc/<pid>/queues/%u failed", in kfd_procfs_add_queue()
507 q->properties.queue_id); in kfd_procfs_add_queue()
508 kobject_put(&q->kobj); in kfd_procfs_add_queue()
523 attr->name = name; in kfd_sysfs_create_file()
524 attr->mode = KFD_SYSFS_FILE_MODE; in kfd_sysfs_create_file()
529 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret); in kfd_sysfs_create_file()
538 if (!p || !p->kobj) in kfd_procfs_add_sysfs_stats()
543 * - proc/<pid>/stats_<gpuid>/ in kfd_procfs_add_sysfs_stats()
544 * - proc/<pid>/stats_<gpuid>/evicted_ms in kfd_procfs_add_sysfs_stats()
545 * - proc/<pid>/stats_<gpuid>/cu_occupancy in kfd_procfs_add_sysfs_stats()
547 for (i = 0; i < p->n_pdds; i++) { in kfd_procfs_add_sysfs_stats()
548 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_stats()
551 "stats_%u", pdd->dev->id); in kfd_procfs_add_sysfs_stats()
552 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
553 if (!pdd->kobj_stats) in kfd_procfs_add_sysfs_stats()
556 ret = kobject_init_and_add(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
558 p->kobj, in kfd_procfs_add_sysfs_stats()
562 pr_warn("Creating KFD proc/stats_%s folder failed", in kfd_procfs_add_sysfs_stats()
564 kobject_put(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
565 pdd->kobj_stats = NULL; in kfd_procfs_add_sysfs_stats()
569 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict, in kfd_procfs_add_sysfs_stats()
572 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_procfs_add_sysfs_stats()
573 kfd_sysfs_create_file(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
574 &pdd->attr_cu_occupancy, in kfd_procfs_add_sysfs_stats()
585 if (!p || !p->kobj) in kfd_procfs_add_sysfs_counters()
590 * - proc/<pid>/counters_<gpuid>/ in kfd_procfs_add_sysfs_counters()
591 * - proc/<pid>/counters_<gpuid>/faults in kfd_procfs_add_sysfs_counters()
592 * - proc/<pid>/counters_<gpuid>/page_in in kfd_procfs_add_sysfs_counters()
593 * - proc/<pid>/counters_<gpuid>/page_out in kfd_procfs_add_sysfs_counters()
595 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) { in kfd_procfs_add_sysfs_counters()
596 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_counters()
600 "counters_%u", pdd->dev->id); in kfd_procfs_add_sysfs_counters()
606 p->kobj, counters_dir_filename); in kfd_procfs_add_sysfs_counters()
608 pr_warn("Creating KFD proc/%s folder failed", in kfd_procfs_add_sysfs_counters()
614 pdd->kobj_counters = kobj_counters; in kfd_procfs_add_sysfs_counters()
615 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults, in kfd_procfs_add_sysfs_counters()
617 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in, in kfd_procfs_add_sysfs_counters()
619 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out, in kfd_procfs_add_sysfs_counters()
628 if (!p || !p->kobj) in kfd_procfs_add_sysfs_files()
633 * - proc/<pid>/vram_<gpuid> in kfd_procfs_add_sysfs_files()
634 * - proc/<pid>/sdma_<gpuid> in kfd_procfs_add_sysfs_files()
636 for (i = 0; i < p->n_pdds; i++) { in kfd_procfs_add_sysfs_files()
637 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_files()
639 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", in kfd_procfs_add_sysfs_files()
640 pdd->dev->id); in kfd_procfs_add_sysfs_files()
641 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram, in kfd_procfs_add_sysfs_files()
642 pdd->vram_filename); in kfd_procfs_add_sysfs_files()
644 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", in kfd_procfs_add_sysfs_files()
645 pdd->dev->id); in kfd_procfs_add_sysfs_files()
646 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma, in kfd_procfs_add_sysfs_files()
647 pdd->sdma_filename); in kfd_procfs_add_sysfs_files()
656 kobject_del(&q->kobj); in kfd_procfs_del_queue()
657 kobject_put(&q->kobj); in kfd_procfs_del_queue()
669 return -ENOMEM; in kfd_process_create_wq()
690 struct kfd_dev *dev = pdd->dev; in kfd_process_free_gpuvm()
692 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv); in kfd_process_free_gpuvm()
693 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv, in kfd_process_free_gpuvm()
697 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
701 * not need to take p->mutex.
707 struct kfd_dev *kdev = pdd->dev; in kfd_process_alloc_gpuvm()
712 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size, in kfd_process_alloc_gpuvm()
713 pdd->drm_priv, &mem, NULL, flags); in kfd_process_alloc_gpuvm()
717 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, in kfd_process_alloc_gpuvm()
718 pdd->drm_priv, NULL); in kfd_process_alloc_gpuvm()
722 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true); in kfd_process_alloc_gpuvm()
730 * We do not need to take p->mutex, because the process is just in kfd_process_alloc_gpuvm()
741 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd, in kfd_process_alloc_gpuvm()
759 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv, in kfd_process_alloc_gpuvm()
766 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
774 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_reserve_ib_mem()
782 if (qpd->ib_kaddr || !qpd->ib_base) in kfd_process_device_reserve_ib_mem()
786 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, in kfd_process_device_reserve_ib_mem()
791 qpd->ib_kaddr = kaddr; in kfd_process_device_reserve_ib_mem()
802 if (!thread->mm) in kfd_create_process()
803 return ERR_PTR(-EINVAL); in kfd_create_process()
806 if (thread->group_leader->mm != thread->mm) in kfd_create_process()
807 return ERR_PTR(-EINVAL); in kfd_create_process()
832 process->kobj = kfd_alloc_struct(process->kobj); in kfd_create_process()
833 if (!process->kobj) { in kfd_create_process()
837 ret = kobject_init_and_add(process->kobj, &procfs_type, in kfd_create_process()
839 (int)process->lead_thread->pid); in kfd_create_process()
842 kobject_put(process->kobj); in kfd_create_process()
846 kfd_sysfs_create_file(process->kobj, &process->attr_pasid, in kfd_create_process()
849 process->kobj_queues = kobject_create_and_add("queues", in kfd_create_process()
850 process->kobj); in kfd_create_process()
851 if (!process->kobj_queues) in kfd_create_process()
852 pr_warn("Creating KFD proc/queues folder failed"); in kfd_create_process()
860 kref_get(&process->ref); in kfd_create_process()
866 hash_del_rcu(&process->kfd_processes); in kfd_create_process()
870 mmu_notifier_put(&process->mmu_notifier); in kfd_create_process()
878 if (!thread->mm) in kfd_get_process()
879 return ERR_PTR(-EINVAL); in kfd_get_process()
882 if (thread->group_leader->mm != thread->mm) in kfd_get_process()
883 return ERR_PTR(-EINVAL); in kfd_get_process()
887 return ERR_PTR(-EINVAL); in kfd_get_process()
898 if (process->mm == mm) in find_process_by_mm()
910 p = find_process_by_mm(thread->mm); in find_process()
918 kref_put(&p->ref, kfd_process_ref_release); in kfd_unref_process()
924 struct kfd_process *p = pdd->process; in kfd_process_device_free_bos()
926 int id; in kfd_process_device_free_bos() local
933 idr_for_each_entry(&pdd->alloc_idr, mem, id) { in kfd_process_device_free_bos()
935 for (i = 0; i < p->n_pdds; i++) { in kfd_process_device_free_bos()
936 struct kfd_process_device *peer_pdd = p->pdds[i]; in kfd_process_device_free_bos()
938 if (!peer_pdd->drm_priv) in kfd_process_device_free_bos()
941 peer_pdd->dev->kgd, mem, peer_pdd->drm_priv); in kfd_process_device_free_bos()
944 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, in kfd_process_device_free_bos()
945 pdd->drm_priv, NULL); in kfd_process_device_free_bos()
946 kfd_process_device_remove_obj_handle(pdd, id); in kfd_process_device_free_bos()
954 for (i = 0; i < p->n_pdds; i++) in kfd_process_free_outstanding_kfd_bos()
955 kfd_process_device_free_bos(p->pdds[i]); in kfd_process_free_outstanding_kfd_bos()
962 for (i = 0; i < p->n_pdds; i++) { in kfd_process_destroy_pdds()
963 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_destroy_pdds()
965 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n", in kfd_process_destroy_pdds()
966 pdd->dev->id, p->pasid); in kfd_process_destroy_pdds()
968 if (pdd->drm_file) { in kfd_process_destroy_pdds()
970 pdd->dev->kgd, pdd->drm_priv); in kfd_process_destroy_pdds()
971 fput(pdd->drm_file); in kfd_process_destroy_pdds()
974 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) in kfd_process_destroy_pdds()
975 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, in kfd_process_destroy_pdds()
978 kfree(pdd->qpd.doorbell_bitmap); in kfd_process_destroy_pdds()
979 idr_destroy(&pdd->alloc_idr); in kfd_process_destroy_pdds()
981 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index); in kfd_process_destroy_pdds()
987 if (pdd->runtime_inuse) { in kfd_process_destroy_pdds()
988 pm_runtime_mark_last_busy(pdd->dev->ddev->dev); in kfd_process_destroy_pdds()
989 pm_runtime_put_autosuspend(pdd->dev->ddev->dev); in kfd_process_destroy_pdds()
990 pdd->runtime_inuse = false; in kfd_process_destroy_pdds()
994 p->pdds[i] = NULL; in kfd_process_destroy_pdds()
996 p->n_pdds = 0; in kfd_process_destroy_pdds()
1004 if (!p->kobj) in kfd_process_remove_sysfs()
1007 sysfs_remove_file(p->kobj, &p->attr_pasid); in kfd_process_remove_sysfs()
1008 kobject_del(p->kobj_queues); in kfd_process_remove_sysfs()
1009 kobject_put(p->kobj_queues); in kfd_process_remove_sysfs()
1010 p->kobj_queues = NULL; in kfd_process_remove_sysfs()
1012 for (i = 0; i < p->n_pdds; i++) { in kfd_process_remove_sysfs()
1013 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1015 sysfs_remove_file(p->kobj, &pdd->attr_vram); in kfd_process_remove_sysfs()
1016 sysfs_remove_file(p->kobj, &pdd->attr_sdma); in kfd_process_remove_sysfs()
1018 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); in kfd_process_remove_sysfs()
1019 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_process_remove_sysfs()
1020 sysfs_remove_file(pdd->kobj_stats, in kfd_process_remove_sysfs()
1021 &pdd->attr_cu_occupancy); in kfd_process_remove_sysfs()
1022 kobject_del(pdd->kobj_stats); in kfd_process_remove_sysfs()
1023 kobject_put(pdd->kobj_stats); in kfd_process_remove_sysfs()
1024 pdd->kobj_stats = NULL; in kfd_process_remove_sysfs()
1027 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) { in kfd_process_remove_sysfs()
1028 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1030 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults); in kfd_process_remove_sysfs()
1031 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in); in kfd_process_remove_sysfs()
1032 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out); in kfd_process_remove_sysfs()
1033 kobject_del(pdd->kobj_counters); in kfd_process_remove_sysfs()
1034 kobject_put(pdd->kobj_counters); in kfd_process_remove_sysfs()
1035 pdd->kobj_counters = NULL; in kfd_process_remove_sysfs()
1038 kobject_del(p->kobj); in kfd_process_remove_sysfs()
1039 kobject_put(p->kobj); in kfd_process_remove_sysfs()
1040 p->kobj = NULL; in kfd_process_remove_sysfs()
1059 dma_fence_put(p->ef); in kfd_process_wq_release()
1063 kfd_pasid_free(p->pasid); in kfd_process_wq_release()
1064 mutex_destroy(&p->mutex); in kfd_process_wq_release()
1066 put_task_struct(p->lead_thread); in kfd_process_wq_release()
1075 INIT_WORK(&p->release_work, kfd_process_wq_release); in kfd_process_ref_release()
1076 queue_work(kfd_process_wq, &p->release_work); in kfd_process_ref_release()
1086 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH); in kfd_process_alloc_notifier()
1105 if (WARN_ON(p->mm != mm)) in kfd_process_notifier_release()
1109 hash_del_rcu(&p->kfd_processes); in kfd_process_notifier_release()
1113 cancel_delayed_work_sync(&p->eviction_work); in kfd_process_notifier_release()
1114 cancel_delayed_work_sync(&p->restore_work); in kfd_process_notifier_release()
1115 cancel_delayed_work_sync(&p->svms.restore_work); in kfd_process_notifier_release()
1117 mutex_lock(&p->mutex); in kfd_process_notifier_release()
1123 for (i = 0; i < p->n_pdds; i++) { in kfd_process_notifier_release()
1124 struct kfd_dev *dev = p->pdds[i]->dev; in kfd_process_notifier_release()
1127 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) { in kfd_process_notifier_release()
1128 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) { in kfd_process_notifier_release()
1129 kfd_dbgmgr_destroy(dev->dbgmgr); in kfd_process_notifier_release()
1130 dev->dbgmgr = NULL; in kfd_process_notifier_release()
1137 pqm_uninit(&p->pqm); in kfd_process_notifier_release()
1140 p->mm = NULL; in kfd_process_notifier_release()
1145 dma_fence_signal(p->ef); in kfd_process_notifier_release()
1147 mutex_unlock(&p->mutex); in kfd_process_notifier_release()
1149 mmu_notifier_put(&p->mmu_notifier); in kfd_process_notifier_release()
1163 for (i = 0; i < p->n_pdds; i++) { in kfd_process_init_cwsr_apu()
1164 struct kfd_dev *dev = p->pdds[i]->dev; in kfd_process_init_cwsr_apu()
1165 struct qcm_process_device *qpd = &p->pdds[i]->qpd; in kfd_process_init_cwsr_apu()
1167 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base) in kfd_process_init_cwsr_apu()
1170 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id); in kfd_process_init_cwsr_apu()
1171 qpd->tba_addr = (int64_t)vm_mmap(filep, 0, in kfd_process_init_cwsr_apu()
1175 if (IS_ERR_VALUE(qpd->tba_addr)) { in kfd_process_init_cwsr_apu()
1176 int err = qpd->tba_addr; in kfd_process_init_cwsr_apu()
1179 qpd->tba_addr = 0; in kfd_process_init_cwsr_apu()
1180 qpd->cwsr_kaddr = NULL; in kfd_process_init_cwsr_apu()
1184 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); in kfd_process_init_cwsr_apu()
1186 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; in kfd_process_init_cwsr_apu()
1188 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); in kfd_process_init_cwsr_apu()
1196 struct kfd_dev *dev = pdd->dev; in kfd_process_device_init_cwsr_dgpu()
1197 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_init_cwsr_dgpu()
1204 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base) in kfd_process_device_init_cwsr_dgpu()
1208 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, in kfd_process_device_init_cwsr_dgpu()
1213 qpd->cwsr_kaddr = kaddr; in kfd_process_device_init_cwsr_dgpu()
1214 qpd->tba_addr = qpd->cwsr_base; in kfd_process_device_init_cwsr_dgpu()
1216 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); in kfd_process_device_init_cwsr_dgpu()
1218 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; in kfd_process_device_init_cwsr_dgpu()
1220 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); in kfd_process_device_init_cwsr_dgpu()
1229 if (qpd->cwsr_kaddr) { in kfd_process_set_trap_handler()
1230 /* KFD trap handler is bound, record as second-level TBA/TMA in kfd_process_set_trap_handler()
1231 * in first-level TMA. First-level trap will jump to second. in kfd_process_set_trap_handler()
1234 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET); in kfd_process_set_trap_handler()
1238 /* No trap handler bound, bind as first-level TBA/TMA. */ in kfd_process_set_trap_handler()
1239 qpd->tba_addr = tba_addr; in kfd_process_set_trap_handler()
1240 qpd->tma_addr = tma_addr; in kfd_process_set_trap_handler()
1256 * built for XNACK-off. On GFXv9 it may perform slower. in kfd_process_xnack_mode()
1258 * Therefore applications built for XNACK-off can always be in kfd_process_xnack_mode()
1262 for (i = 0; i < p->n_pdds; i++) { in kfd_process_xnack_mode()
1263 struct kfd_dev *dev = p->pdds[i]->dev; in kfd_process_xnack_mode()
1269 if (dev->device_info->asic_family < CHIP_VEGA10) in kfd_process_xnack_mode()
1272 * per-process XNACK mode selection. But let the dev->noretry in kfd_process_xnack_mode()
1276 dev->device_info->asic_family == CHIP_ALDEBARAN) in kfd_process_xnack_mode()
1281 * management and memory-manager-related preemptions or in kfd_process_xnack_mode()
1284 if (dev->device_info->asic_family >= CHIP_NAVI10) in kfd_process_xnack_mode()
1287 if (dev->noretry) in kfd_process_xnack_mode()
1302 int err = -ENOMEM; in create_process()
1308 kref_init(&process->ref); in create_process()
1309 mutex_init(&process->mutex); in create_process()
1310 process->mm = thread->mm; in create_process()
1311 process->lead_thread = thread->group_leader; in create_process()
1312 process->n_pdds = 0; in create_process()
1313 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); in create_process()
1314 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); in create_process()
1315 process->last_restore_timestamp = get_jiffies_64(); in create_process()
1317 process->is_32bit_user_mode = in_compat_syscall(); in create_process()
1319 process->pasid = kfd_pasid_alloc(); in create_process()
1320 if (process->pasid == 0) in create_process()
1323 err = pqm_init(&process->pqm, process); in create_process()
1333 process->xnack_enabled = kfd_process_xnack_mode(process, false); in create_process()
1340 hash_add_rcu(kfd_processes_table, &process->kfd_processes, in create_process()
1341 (uintptr_t)process->mm); in create_process()
1348 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm); in create_process()
1353 BUG_ON(mn != &process->mmu_notifier); in create_process()
1355 get_task_struct(process->lead_thread); in create_process()
1360 hash_del_rcu(&process->kfd_processes); in create_process()
1366 pqm_uninit(&process->pqm); in create_process()
1368 kfd_pasid_free(process->pasid); in create_process()
1370 mutex_destroy(&process->mutex); in create_process()
1380 int range_start = dev->shared_resources.non_cp_doorbells_start; in init_doorbell_bitmap()
1381 int range_end = dev->shared_resources.non_cp_doorbells_end; in init_doorbell_bitmap()
1383 if (!KFD_IS_SOC15(dev->device_info->asic_family)) in init_doorbell_bitmap()
1386 qpd->doorbell_bitmap = in init_doorbell_bitmap()
1389 if (!qpd->doorbell_bitmap) in init_doorbell_bitmap()
1390 return -ENOMEM; in init_doorbell_bitmap()
1393 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end); in init_doorbell_bitmap()
1394 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", in init_doorbell_bitmap()
1400 set_bit(i, qpd->doorbell_bitmap); in init_doorbell_bitmap()
1402 qpd->doorbell_bitmap); in init_doorbell_bitmap()
1414 for (i = 0; i < p->n_pdds; i++) in kfd_get_process_device_data()
1415 if (p->pdds[i]->dev == dev) in kfd_get_process_device_data()
1416 return p->pdds[i]; in kfd_get_process_device_data()
1426 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE)) in kfd_create_process_device_data()
1432 if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) { in kfd_create_process_device_data()
1437 if (init_doorbell_bitmap(&pdd->qpd, dev)) { in kfd_create_process_device_data()
1442 pdd->dev = dev; in kfd_create_process_device_data()
1443 INIT_LIST_HEAD(&pdd->qpd.queues_list); in kfd_create_process_device_data()
1444 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); in kfd_create_process_device_data()
1445 pdd->qpd.dqm = dev->dqm; in kfd_create_process_device_data()
1446 pdd->qpd.pqm = &p->pqm; in kfd_create_process_device_data()
1447 pdd->qpd.evicted = 0; in kfd_create_process_device_data()
1448 pdd->qpd.mapped_gws_queue = false; in kfd_create_process_device_data()
1449 pdd->process = p; in kfd_create_process_device_data()
1450 pdd->bound = PDD_UNBOUND; in kfd_create_process_device_data()
1451 pdd->already_dequeued = false; in kfd_create_process_device_data()
1452 pdd->runtime_inuse = false; in kfd_create_process_device_data()
1453 pdd->vram_usage = 0; in kfd_create_process_device_data()
1454 pdd->sdma_past_activity_counter = 0; in kfd_create_process_device_data()
1455 atomic64_set(&pdd->evict_duration_counter, 0); in kfd_create_process_device_data()
1456 p->pdds[p->n_pdds++] = pdd; in kfd_create_process_device_data()
1459 idr_init(&pdd->alloc_idr); in kfd_create_process_device_data()
1469 * kfd_process_device_init_vm - Initialize a VM for a process-device
1471 * @pdd: The process-device
1480 * Returns 0 on success, -errno on failure.
1490 return -EINVAL; in kfd_process_device_init_vm()
1492 if (pdd->drm_priv) in kfd_process_device_init_vm()
1493 return -EBUSY; in kfd_process_device_init_vm()
1495 p = pdd->process; in kfd_process_device_init_vm()
1496 dev = pdd->dev; in kfd_process_device_init_vm()
1499 dev->kgd, drm_file, p->pasid, in kfd_process_device_init_vm()
1500 &p->kgd_process_info, &p->ef); in kfd_process_device_init_vm()
1505 pdd->drm_priv = drm_file->private_data; in kfd_process_device_init_vm()
1514 pdd->drm_file = drm_file; in kfd_process_device_init_vm()
1521 pdd->drm_priv = NULL; in kfd_process_device_init_vm()
1527 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1542 return ERR_PTR(-ENOMEM); in kfd_bind_process_to_device()
1545 if (!pdd->drm_priv) in kfd_bind_process_to_device()
1546 return ERR_PTR(-ENODEV); in kfd_bind_process_to_device()
1549 * signal runtime-pm system to auto resume and prevent in kfd_bind_process_to_device()
1553 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1554 err = pm_runtime_get_sync(dev->ddev->dev); in kfd_bind_process_to_device()
1556 pm_runtime_put_autosuspend(dev->ddev->dev); in kfd_bind_process_to_device()
1569 pdd->runtime_inuse = true; in kfd_bind_process_to_device()
1575 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1576 pm_runtime_mark_last_busy(dev->ddev->dev); in kfd_bind_process_to_device()
1577 pm_runtime_put_autosuspend(dev->ddev->dev); in kfd_bind_process_to_device()
1589 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); in kfd_process_device_create_obj_handle()
1601 return idr_find(&pdd->alloc_idr, handle); in kfd_process_device_translate_handle()
1611 idr_remove(&pdd->alloc_idr, handle); in kfd_process_device_remove_obj_handle()
1614 /* This increments the process->ref counter. */
1623 if (p->pasid == pasid) { in kfd_lookup_process_by_pasid()
1624 kref_get(&p->ref); in kfd_lookup_process_by_pasid()
1635 /* This increments the process->ref counter. */
1644 kref_get(&p->ref); in kfd_lookup_process_by_mm()
1651 /* kfd_process_evict_queues - Evict all user queues of a process
1653 * Eviction is reference-counted per process-device. This means multiple
1662 for (i = 0; i < p->n_pdds; i++) { in kfd_process_evict_queues()
1663 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues()
1665 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1666 &pdd->qpd); in kfd_process_evict_queues()
1680 for (i = 0; i < p->n_pdds; i++) { in kfd_process_evict_queues()
1681 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues()
1685 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1686 &pdd->qpd)) in kfd_process_evict_queues()
1689 n_evicted--; in kfd_process_evict_queues()
1695 /* kfd_process_restore_queues - Restore all user queues of a process */
1701 for (i = 0; i < p->n_pdds; i++) { in kfd_process_restore_queues()
1702 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_restore_queues()
1704 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_restore_queues()
1705 &pdd->qpd); in kfd_process_restore_queues()
1720 for (i = 0; i < p->n_pdds; i++) in kfd_process_gpuidx_from_gpuid()
1721 if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id) in kfd_process_gpuidx_from_gpuid()
1723 return -EINVAL; in kfd_process_gpuidx_from_gpuid()
1733 for (i = 0; i < p->n_pdds; i++) in kfd_process_gpuid_from_kgd()
1734 if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) { in kfd_process_gpuid_from_kgd()
1735 *gpuid = p->pdds[i]->dev->id; in kfd_process_gpuid_from_kgd()
1739 return -EINVAL; in kfd_process_gpuid_from_kgd()
1754 WARN_ONCE(p->last_eviction_seqno != p->ef->seqno, in evict_process_worker()
1763 flush_delayed_work(&p->restore_work); in evict_process_worker()
1765 pr_debug("Started evicting pasid 0x%x\n", p->pasid); in evict_process_worker()
1768 dma_fence_signal(p->ef); in evict_process_worker()
1769 dma_fence_put(p->ef); in evict_process_worker()
1770 p->ef = NULL; in evict_process_worker()
1771 queue_delayed_work(kfd_restore_wq, &p->restore_work, in evict_process_worker()
1774 pr_debug("Finished evicting pasid 0x%x\n", p->pasid); in evict_process_worker()
1776 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid); in evict_process_worker()
1791 pr_debug("Started restoring pasid 0x%x\n", p->pasid); in restore_process_worker()
1799 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two in restore_process_worker()
1803 p->last_restore_timestamp = get_jiffies_64(); in restore_process_worker()
1804 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info, in restore_process_worker()
1805 &p->ef); in restore_process_worker()
1808 p->pasid, PROCESS_BACK_OFF_TIME_MS); in restore_process_worker()
1809 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work, in restore_process_worker()
1817 pr_debug("Finished restoring pasid 0x%x\n", p->pasid); in restore_process_worker()
1819 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid); in restore_process_worker()
1830 cancel_delayed_work_sync(&p->eviction_work); in kfd_suspend_all_processes()
1831 cancel_delayed_work_sync(&p->restore_work); in kfd_suspend_all_processes()
1834 pr_err("Failed to suspend process 0x%x\n", p->pasid); in kfd_suspend_all_processes()
1835 dma_fence_signal(p->ef); in kfd_suspend_all_processes()
1836 dma_fence_put(p->ef); in kfd_suspend_all_processes()
1837 p->ef = NULL; in kfd_suspend_all_processes()
1849 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) { in kfd_resume_all_processes()
1851 p->pasid); in kfd_resume_all_processes()
1852 ret = -EFAULT; in kfd_resume_all_processes()
1865 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) { in kfd_reserved_mem_mmap()
1867 return -EINVAL; in kfd_reserved_mem_mmap()
1872 return -EINVAL; in kfd_reserved_mem_mmap()
1873 qpd = &pdd->qpd; in kfd_reserved_mem_mmap()
1875 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in kfd_reserved_mem_mmap()
1877 if (!qpd->cwsr_kaddr) { in kfd_reserved_mem_mmap()
1879 return -ENOMEM; in kfd_reserved_mem_mmap()
1882 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND in kfd_reserved_mem_mmap()
1885 return remap_pfn_range(vma, vma->vm_start, in kfd_reserved_mem_mmap()
1886 PFN_DOWN(__pa(qpd->cwsr_kaddr)), in kfd_reserved_mem_mmap()
1887 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot); in kfd_reserved_mem_mmap()
1892 struct kfd_dev *dev = pdd->dev; in kfd_flush_tlb()
1894 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { in kfd_flush_tlb()
1898 if (pdd->qpd.vmid) in kfd_flush_tlb()
1899 amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd, in kfd_flush_tlb()
1900 pdd->qpd.vmid); in kfd_flush_tlb()
1902 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd, in kfd_flush_tlb()
1903 pdd->process->pasid, type); in kfd_flush_tlb()
1919 p->lead_thread->tgid, p->pasid); in kfd_debugfs_mqds_by_process()
1921 mutex_lock(&p->mutex); in kfd_debugfs_mqds_by_process()
1922 r = pqm_debugfs_mqds(m, &p->pqm); in kfd_debugfs_mqds_by_process()
1923 mutex_unlock(&p->mutex); in kfd_debugfs_mqds_by_process()