104 	struct qcm_process_device *qpd;  in kfd_sdma_activity_worker()  local
117 	qpd = &pdd->qpd;  in kfd_sdma_activity_worker()
118 	if (!dqm || !qpd)  in kfd_sdma_activity_worker()
149 	list_for_each_entry(q, &qpd->queues_list, list) {  in kfd_sdma_activity_worker()
211 	list_for_each_entry(q, &qpd->queues_list, list) {  in kfd_sdma_activity_worker()
280 	if (pdd->qpd.queue_count == 0) {  in kfd_get_cu_occupancy()
774 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_reserve_ib_mem()  local
782 	if (qpd->ib_kaddr || !qpd->ib_base)  in kfd_process_device_reserve_ib_mem()
786 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,  in kfd_process_device_reserve_ib_mem()
791 	qpd->ib_kaddr = kaddr;  in kfd_process_device_reserve_ib_mem()
974 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)  in kfd_process_destroy_pdds()
975 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,  in kfd_process_destroy_pdds()
978 		kfree(pdd->qpd.doorbell_bitmap);  in kfd_process_destroy_pdds()
1165 		struct qcm_process_device *qpd = &p->pdds[i]->qpd;  in kfd_process_init_cwsr_apu()  local
1167 		if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)  in kfd_process_init_cwsr_apu()
1171 		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,  in kfd_process_init_cwsr_apu()
1175 		if (IS_ERR_VALUE(qpd->tba_addr)) {  in kfd_process_init_cwsr_apu()
1176 			int err = qpd->tba_addr;  in kfd_process_init_cwsr_apu()
1179 			qpd->tba_addr = 0;  in kfd_process_init_cwsr_apu()
1180 			qpd->cwsr_kaddr = NULL;  in kfd_process_init_cwsr_apu()
1184 		memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);  in kfd_process_init_cwsr_apu()
1186 		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;  in kfd_process_init_cwsr_apu()
1188 			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);  in kfd_process_init_cwsr_apu()
1197 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_init_cwsr_dgpu()  local
1204 	if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)  in kfd_process_device_init_cwsr_dgpu()
1208 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,  in kfd_process_device_init_cwsr_dgpu()
1213 	qpd->cwsr_kaddr = kaddr;  in kfd_process_device_init_cwsr_dgpu()
1214 	qpd->tba_addr = qpd->cwsr_base;  in kfd_process_device_init_cwsr_dgpu()
1216 	memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);  in kfd_process_device_init_cwsr_dgpu()
1218 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;  in kfd_process_device_init_cwsr_dgpu()
1220 		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);  in kfd_process_device_init_cwsr_dgpu()
1225 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,  in kfd_process_set_trap_handler()  argument
1229 	if (qpd->cwsr_kaddr) {  in kfd_process_set_trap_handler()
1234 			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);  in kfd_process_set_trap_handler()
1239 		qpd->tba_addr = tba_addr;  in kfd_process_set_trap_handler()
1240 		qpd->tma_addr = tma_addr;  in kfd_process_set_trap_handler()
1376 static int init_doorbell_bitmap(struct qcm_process_device *qpd,  in init_doorbell_bitmap()  argument
1386 	qpd->doorbell_bitmap =  in init_doorbell_bitmap()
1389 	if (!qpd->doorbell_bitmap)  in init_doorbell_bitmap()
1400 			set_bit(i, qpd->doorbell_bitmap);  in init_doorbell_bitmap()
1402 				qpd->doorbell_bitmap);  in init_doorbell_bitmap()
1437 	if (init_doorbell_bitmap(&pdd->qpd, dev)) {  in kfd_create_process_device_data()
1443 	INIT_LIST_HEAD(&pdd->qpd.queues_list);  in kfd_create_process_device_data()
1444 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);  in kfd_create_process_device_data()
1445 	pdd->qpd.dqm = dev->dqm;  in kfd_create_process_device_data()
1446 	pdd->qpd.pqm = &p->pqm;  in kfd_create_process_device_data()
1447 	pdd->qpd.evicted = 0;  in kfd_create_process_device_data()
1448 	pdd->qpd.mapped_gws_queue = false;  in kfd_create_process_device_data()
1666 							    &pdd->qpd);  in kfd_process_evict_queues()
1686 							      &pdd->qpd))  in kfd_process_evict_queues()
1705 							      &pdd->qpd);  in kfd_process_restore_queues()
1863 	struct qcm_process_device *qpd;  in kfd_reserved_mem_mmap()  local
1873 	qpd = &pdd->qpd;  in kfd_reserved_mem_mmap()
1875 	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,  in kfd_reserved_mem_mmap()
1877 	if (!qpd->cwsr_kaddr) {  in kfd_reserved_mem_mmap()
1886 			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),  in kfd_reserved_mem_mmap()
1898 		if (pdd->qpd.vmid)  in kfd_flush_tlb()
1900 							pdd->qpd.vmid);  in kfd_flush_tlb()