106 	struct qcm_process_device *qpd;  in kfd_sdma_activity_worker()  local
119 	qpd = &pdd->qpd;  in kfd_sdma_activity_worker()
120 	if (!dqm || !qpd)  in kfd_sdma_activity_worker()
151 	list_for_each_entry(q, &qpd->queues_list, list) {  in kfd_sdma_activity_worker()
213 	list_for_each_entry(q, &qpd->queues_list, list) {  in kfd_sdma_activity_worker()
281 	if (pdd->qpd.queue_count == 0) {  in kfd_get_cu_occupancy()
765 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_reserve_ib_mem()  local
774 	if (qpd->ib_kaddr || !qpd->ib_base)  in kfd_process_device_reserve_ib_mem()
778 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,  in kfd_process_device_reserve_ib_mem()
783 	qpd->ib_mem = mem;  in kfd_process_device_reserve_ib_mem()
784 	qpd->ib_kaddr = kaddr;  in kfd_process_device_reserve_ib_mem()
791 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_destroy_ib_mem()  local
793 	if (!qpd->ib_kaddr || !qpd->ib_base)  in kfd_process_device_destroy_ib_mem()
796 	kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);  in kfd_process_device_destroy_ib_mem()
1032 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)  in kfd_process_destroy_pdds()
1033 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,  in kfd_process_destroy_pdds()
1289 		struct qcm_process_device *qpd = &p->pdds[i]->qpd;  in kfd_process_init_cwsr_apu()  local
1291 		if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)  in kfd_process_init_cwsr_apu()
1295 		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,  in kfd_process_init_cwsr_apu()
1299 		if (IS_ERR_VALUE(qpd->tba_addr)) {  in kfd_process_init_cwsr_apu()
1300 			int err = qpd->tba_addr;  in kfd_process_init_cwsr_apu()
1303 			qpd->tba_addr = 0;  in kfd_process_init_cwsr_apu()
1304 			qpd->cwsr_kaddr = NULL;  in kfd_process_init_cwsr_apu()
1308 		memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);  in kfd_process_init_cwsr_apu()
1310 		kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);  in kfd_process_init_cwsr_apu()
1312 		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;  in kfd_process_init_cwsr_apu()
1314 			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);  in kfd_process_init_cwsr_apu()
1325 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_init_cwsr_dgpu()  local
1333 	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)  in kfd_process_device_init_cwsr_dgpu()
1337 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,  in kfd_process_device_init_cwsr_dgpu()
1342 	qpd->cwsr_mem = mem;  in kfd_process_device_init_cwsr_dgpu()
1343 	qpd->cwsr_kaddr = kaddr;  in kfd_process_device_init_cwsr_dgpu()
1344 	qpd->tba_addr = qpd->cwsr_base;  in kfd_process_device_init_cwsr_dgpu()
1346 	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);  in kfd_process_device_init_cwsr_dgpu()
1348 	kfd_process_set_trap_debug_flag(&pdd->qpd,  in kfd_process_device_init_cwsr_dgpu()
1351 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;  in kfd_process_device_init_cwsr_dgpu()
1353 		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);  in kfd_process_device_init_cwsr_dgpu()
1361 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_destroy_cwsr_dgpu()  local
1363 	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)  in kfd_process_device_destroy_cwsr_dgpu()
1366 	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);  in kfd_process_device_destroy_cwsr_dgpu()
1369 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,  in kfd_process_set_trap_handler()  argument
1373 	if (qpd->cwsr_kaddr) {  in kfd_process_set_trap_handler()
1378 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);  in kfd_process_set_trap_handler()
1383 		qpd->tba_addr = tba_addr;  in kfd_process_set_trap_handler()
1384 		qpd->tma_addr = tma_addr;  in kfd_process_set_trap_handler()
1437 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,  in kfd_process_set_trap_debug_flag()  argument
1440 	if (qpd->cwsr_kaddr) {  in kfd_process_set_trap_debug_flag()
1442 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);  in kfd_process_set_trap_debug_flag()
1574 	INIT_LIST_HEAD(&pdd->qpd.queues_list);  in kfd_create_process_device_data()
1575 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);  in kfd_create_process_device_data()
1576 	pdd->qpd.dqm = dev->dqm;  in kfd_create_process_device_data()
1577 	pdd->qpd.pqm = &p->pqm;  in kfd_create_process_device_data()
1578 	pdd->qpd.evicted = 0;  in kfd_create_process_device_data()
1579 	pdd->qpd.mapped_gws_queue = false;  in kfd_create_process_device_data()
1823 							    &pdd->qpd);  in kfd_process_evict_queues()
1850 							      &pdd->qpd))  in kfd_process_evict_queues()
1871 							      &pdd->qpd);  in kfd_process_restore_queues()
2030 	struct qcm_process_device *qpd;  in kfd_reserved_mem_mmap()  local
2040 	qpd = &pdd->qpd;  in kfd_reserved_mem_mmap()
2042 	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,  in kfd_reserved_mem_mmap()
2044 	if (!qpd->cwsr_kaddr) {  in kfd_reserved_mem_mmap()
2053 			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),  in kfd_reserved_mem_mmap()
2077 		if (pdd->qpd.vmid)  in kfd_flush_tlb()
2079 							pdd->qpd.vmid);  in kfd_flush_tlb()