40 static bool kq_initialize(struct kernel_queue *kq, struct kfd_dev *dev,  in kq_initialize()  argument
60 	kq->dev = dev;  in kq_initialize()
61 	kq->nop_packet = nop.u32all;  in kq_initialize()
64 		kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_DIQ];  in kq_initialize()
67 		kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];  in kq_initialize()
74 	if (!kq->mqd_mgr)  in kq_initialize()
84 	retval = kfd_gtt_sa_allocate(dev, queue_size, &kq->pq);  in kq_initialize()
90 	kq->pq_kernel_addr = kq->pq->cpu_ptr;  in kq_initialize()
91 	kq->pq_gpu_addr = kq->pq->gpu_addr;  in kq_initialize()
95 		retval = kfd_gtt_sa_allocate(dev, PAGE_SIZE, &kq->eop_mem);  in kq_initialize()
99 		kq->eop_gpu_addr = kq->eop_mem->gpu_addr;  in kq_initialize()
100 		kq->eop_kernel_addr = kq->eop_mem->cpu_ptr;  in kq_initialize()
102 		memset(kq->eop_kernel_addr, 0, PAGE_SIZE);  in kq_initialize()
105 	retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->rptr_kernel),  in kq_initialize()
106 					&kq->rptr_mem);  in kq_initialize()
111 	kq->rptr_kernel = kq->rptr_mem->cpu_ptr;  in kq_initialize()
112 	kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr;  in kq_initialize()
115 					&kq->wptr_mem);  in kq_initialize()
120 	kq->wptr_kernel = kq->wptr_mem->cpu_ptr;  in kq_initialize()
121 	kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr;  in kq_initialize()
123 	memset(kq->pq_kernel_addr, 0, queue_size);  in kq_initialize()
124 	memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel));  in kq_initialize()
125 	memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel));  in kq_initialize()
134 	prop.queue_address = kq->pq_gpu_addr;  in kq_initialize()
135 	prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr;  in kq_initialize()
136 	prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;  in kq_initialize()
137 	prop.eop_ring_buffer_address = kq->eop_gpu_addr;  in kq_initialize()
141 	if (init_queue(&kq->queue, &prop) != 0)  in kq_initialize()
144 	kq->queue->device = dev;  in kq_initialize()
145 	kq->queue->process = kfd_get_process(current);  in kq_initialize()
147 	kq->queue->mqd_mem_obj = kq->mqd_mgr->allocate_mqd(kq->mqd_mgr->dev,  in kq_initialize()
148 					&kq->queue->properties);  in kq_initialize()
149 	if (!kq->queue->mqd_mem_obj)  in kq_initialize()
151 	kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,  in kq_initialize()
152 					kq->queue->mqd_mem_obj,  in kq_initialize()
153 					&kq->queue->gart_mqd_addr,  in kq_initialize()
154 					&kq->queue->properties);  in kq_initialize()
158 		kq->queue->pipe = KFD_CIK_HIQ_PIPE;  in kq_initialize()
159 		kq->queue->queue = KFD_CIK_HIQ_QUEUE;  in kq_initialize()
160 		kq->mqd_mgr->load_mqd(kq->mqd_mgr, kq->queue->mqd,  in kq_initialize()
161 				kq->queue->pipe, kq->queue->queue,  in kq_initialize()
162 				&kq->queue->properties, NULL);  in kq_initialize()
167 						&kq->fence_mem_obj);  in kq_initialize()
172 		kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr;  in kq_initialize()
173 		kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr;  in kq_initialize()
176 	print_queue(kq->queue);  in kq_initialize()
180 	kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd, kq->queue->mqd_mem_obj);  in kq_initialize()
182 	uninit_queue(kq->queue);  in kq_initialize()
184 	kfd_gtt_sa_free(dev, kq->wptr_mem);  in kq_initialize()
186 	kfd_gtt_sa_free(dev, kq->rptr_mem);  in kq_initialize()
188 	kfd_gtt_sa_free(dev, kq->eop_mem);  in kq_initialize()
190 	kfd_gtt_sa_free(dev, kq->pq);  in kq_initialize()
199 static void kq_uninitialize(struct kernel_queue *kq, bool hanging)  in kq_uninitialize()  argument
201 	if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ && !hanging)  in kq_uninitialize()
202 		kq->mqd_mgr->destroy_mqd(kq->mqd_mgr,  in kq_uninitialize()
203 					kq->queue->mqd,  in kq_uninitialize()
206 					kq->queue->pipe,  in kq_uninitialize()
207 					kq->queue->queue);  in kq_uninitialize()
208 	else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)  in kq_uninitialize()
209 		kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);  in kq_uninitialize()
211 	kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd,  in kq_uninitialize()
212 				kq->queue->mqd_mem_obj);  in kq_uninitialize()
214 	kfd_gtt_sa_free(kq->dev, kq->rptr_mem);  in kq_uninitialize()
215 	kfd_gtt_sa_free(kq->dev, kq->wptr_mem);  in kq_uninitialize()
220 	kfd_gtt_sa_free(kq->dev, kq->eop_mem);  in kq_uninitialize()
222 	kfd_gtt_sa_free(kq->dev, kq->pq);  in kq_uninitialize()
223 	kfd_release_kernel_doorbell(kq->dev,  in kq_uninitialize()
224 					kq->queue->properties.doorbell_ptr);  in kq_uninitialize()
225 	uninit_queue(kq->queue);  in kq_uninitialize()
228 int kq_acquire_packet_buffer(struct kernel_queue *kq,  in kq_acquire_packet_buffer()  argument
242 	rptr = *kq->rptr_kernel;  in kq_acquire_packet_buffer()
243 	wptr = kq->pending_wptr;  in kq_acquire_packet_buffer()
244 	wptr64 = kq->pending_wptr64;  in kq_acquire_packet_buffer()
245 	queue_address = (unsigned int *)kq->pq_kernel_addr;  in kq_acquire_packet_buffer()
246 	queue_size_dwords = kq->queue->properties.queue_size / 4;  in kq_acquire_packet_buffer()
272 			queue_address[wptr] = kq->nop_packet;  in kq_acquire_packet_buffer()
279 	kq->pending_wptr = wptr + packet_size_in_dwords;  in kq_acquire_packet_buffer()
280 	kq->pending_wptr64 = wptr64 + packet_size_in_dwords;  in kq_acquire_packet_buffer()
289 void kq_submit_packet(struct kernel_queue *kq)  in kq_submit_packet()  argument
294 	for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) {  in kq_submit_packet()
295 		pr_debug("0x%2X ", kq->pq_kernel_addr[i]);  in kq_submit_packet()
301 	if (kq->dev->device_info->doorbell_size == 8) {  in kq_submit_packet()
302 		*kq->wptr64_kernel = kq->pending_wptr64;  in kq_submit_packet()
303 		write_kernel_doorbell64(kq->queue->properties.doorbell_ptr,  in kq_submit_packet()
304 					kq->pending_wptr64);  in kq_submit_packet()
306 		*kq->wptr_kernel = kq->pending_wptr;  in kq_submit_packet()
307 		write_kernel_doorbell(kq->queue->properties.doorbell_ptr,  in kq_submit_packet()
308 					kq->pending_wptr);  in kq_submit_packet()
312 void kq_rollback_packet(struct kernel_queue *kq)  in kq_rollback_packet()  argument
314 	if (kq->dev->device_info->doorbell_size == 8) {  in kq_rollback_packet()
315 		kq->pending_wptr64 = *kq->wptr64_kernel;  in kq_rollback_packet()
316 		kq->pending_wptr = *kq->wptr_kernel %  in kq_rollback_packet()
317 			(kq->queue->properties.queue_size / 4);  in kq_rollback_packet()
319 		kq->pending_wptr = *kq->wptr_kernel;  in kq_rollback_packet()
326 	struct kernel_queue *kq;  in kernel_queue_init()  local
328 	kq = kzalloc(sizeof(*kq), GFP_KERNEL);  in kernel_queue_init()
329 	if (!kq)  in kernel_queue_init()
332 	if (kq_initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE))  in kernel_queue_init()
333 		return kq;  in kernel_queue_init()
337 	kfree(kq);  in kernel_queue_init()
341 void kernel_queue_uninit(struct kernel_queue *kq, bool hanging)  in kernel_queue_uninit()  argument
343 	kq_uninitialize(kq, hanging);  in kernel_queue_uninit()
344 	kfree(kq);  in kernel_queue_uninit()
350 	struct kernel_queue *kq;  in test_kq()  local
356 	kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ);  in test_kq()
357 	if (unlikely(!kq)) {  in test_kq()
363 	retval = kq_acquire_packet_buffer(kq, 5, &buffer);  in test_kq()
370 		buffer[i] = kq->nop_packet;  in test_kq()
371 	kq_submit_packet(kq);  in test_kq()