27 static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)  in queue_free_slots()  argument
29 	int delta = (q->pi - q->ci);  in queue_free_slots()
40 	struct hl_hw_queue *q;  in hl_int_hw_queue_update_ci()  local
48 	q = &hdev->kernel_queues[0];  in hl_int_hw_queue_update_ci()
49 	for (i = 0 ; i < HL_MAX_QUEUES ; i++, q++) {  in hl_int_hw_queue_update_ci()
50 		if (q->queue_type == QUEUE_TYPE_INT) {  in hl_int_hw_queue_update_ci()
51 			q->ci += cs->jobs_in_queue_cnt[i];  in hl_int_hw_queue_update_ci()
52 			q->ci &= ((q->int_queue_len << 1) - 1);  in hl_int_hw_queue_update_ci()
76 static void ext_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,  in ext_queue_submit_bd()  argument
81 	bd = (struct hl_bd *) (uintptr_t) q->kernel_address;  in ext_queue_submit_bd()
82 	bd += hl_pi_2_offset(q->pi);  in ext_queue_submit_bd()
87 	q->pi = hl_queue_inc_ptr(q->pi);  in ext_queue_submit_bd()
88 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in ext_queue_submit_bd()
110 				struct hl_hw_queue *q, int num_of_entries,  in ext_queue_sanity_checks()  argument
114 			&hdev->completion_queue[q->hw_queue_id].free_slots_cnt;  in ext_queue_sanity_checks()
118 	free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);  in ext_queue_sanity_checks()
122 			q->hw_queue_id, num_of_entries);  in ext_queue_sanity_checks()
136 				num_of_entries, q->hw_queue_id);  in ext_queue_sanity_checks()
159 					struct hl_hw_queue *q,  in int_queue_sanity_checks()  argument
165 	free_slots_cnt = queue_free_slots(q, q->int_queue_len);  in int_queue_sanity_checks()
169 			q->hw_queue_id, num_of_entries);  in int_queue_sanity_checks()
190 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_send_cb_no_cmpl()  local
201 	if (q->queue_type != QUEUE_TYPE_CPU)  in hl_hw_queue_send_cb_no_cmpl()
209 	rc = ext_queue_sanity_checks(hdev, q, 1, false);  in hl_hw_queue_send_cb_no_cmpl()
213 	ext_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);  in hl_hw_queue_send_cb_no_cmpl()
216 	if (q->queue_type != QUEUE_TYPE_CPU)  in hl_hw_queue_send_cb_no_cmpl()
233 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in ext_hw_queue_schedule_job()  local
246 	ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK);  in ext_hw_queue_schedule_job()
253 				((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)  in ext_hw_queue_schedule_job()
265 	cq = &hdev->completion_queue[q->hw_queue_id];  in ext_hw_queue_schedule_job()
271 						q->hw_queue_id);  in ext_hw_queue_schedule_job()
273 	q->shadow_queue[hl_pi_2_offset(q->pi)] = job;  in ext_hw_queue_schedule_job()
277 	ext_queue_submit_bd(hdev, q, ctl, len, ptr);  in ext_hw_queue_schedule_job()
291 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in int_hw_queue_schedule_job()  local
299 	pi = (__le64 *) (uintptr_t) (q->kernel_address +  in int_hw_queue_schedule_job()
300 		((q->pi & (q->int_queue_len - 1)) * sizeof(bd)));  in int_hw_queue_schedule_job()
302 	q->pi++;  in int_hw_queue_schedule_job()
303 	q->pi &= ((q->int_queue_len << 1) - 1);  in int_hw_queue_schedule_job()
307 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in int_hw_queue_schedule_job()
320 	struct hl_hw_queue *q;  in hl_hw_queue_schedule_cs()  local
332 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
334 	for (i = 0, cq_cnt = 0 ; i < HL_MAX_QUEUES ; i++, q++) {  in hl_hw_queue_schedule_cs()
335 		if (q->queue_type == QUEUE_TYPE_EXT) {  in hl_hw_queue_schedule_cs()
337 				rc = ext_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
343 		} else if (q->queue_type == QUEUE_TYPE_INT) {  in hl_hw_queue_schedule_cs()
345 				rc = int_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
387 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
388 	for (i = 0 ; (i < HL_MAX_QUEUES) && (cq_cnt > 0) ; i++, q++) {  in hl_hw_queue_schedule_cs()
389 		if ((q->queue_type == QUEUE_TYPE_EXT) &&  in hl_hw_queue_schedule_cs()
412 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_inc_ci_kernel()  local
414 	q->ci = hl_queue_inc_ptr(q->ci);  in hl_hw_queue_inc_ci_kernel()
418 				struct hl_hw_queue *q, bool is_cpu_queue)  in ext_and_cpu_hw_queue_init()  argument
426 							&q->bus_address);  in ext_and_cpu_hw_queue_init()
430 						&q->bus_address,  in ext_and_cpu_hw_queue_init()
435 	q->kernel_address = (u64) (uintptr_t) p;  in ext_and_cpu_hw_queue_init()
437 	q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH,  in ext_and_cpu_hw_queue_init()
438 					sizeof(*q->shadow_queue),  in ext_and_cpu_hw_queue_init()
440 	if (!q->shadow_queue) {  in ext_and_cpu_hw_queue_init()
443 			q->hw_queue_id);  in ext_and_cpu_hw_queue_init()
449 	q->ci = 0;  in ext_and_cpu_hw_queue_init()
450 	q->pi = 0;  in ext_and_cpu_hw_queue_init()
458 					(void *) (uintptr_t) q->kernel_address);  in ext_and_cpu_hw_queue_init()
462 					(void *) (uintptr_t) q->kernel_address,  in ext_and_cpu_hw_queue_init()
463 					q->bus_address);  in ext_and_cpu_hw_queue_init()
468 static int int_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in int_hw_queue_init()  argument
472 	p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,  in int_hw_queue_init()
473 					&q->bus_address, &q->int_queue_len);  in int_hw_queue_init()
477 			q->hw_queue_id);  in int_hw_queue_init()
481 	q->kernel_address = (u64) (uintptr_t) p;  in int_hw_queue_init()
482 	q->pi = 0;  in int_hw_queue_init()
483 	q->ci = 0;  in int_hw_queue_init()
488 static int cpu_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in cpu_hw_queue_init()  argument
490 	return ext_and_cpu_hw_queue_init(hdev, q, true);  in cpu_hw_queue_init()
493 static int ext_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in ext_hw_queue_init()  argument
495 	return ext_and_cpu_hw_queue_init(hdev, q, false);  in ext_hw_queue_init()
508 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in hw_queue_init()  argument
515 	q->hw_queue_id = hw_queue_id;  in hw_queue_init()
517 	switch (q->queue_type) {  in hw_queue_init()
519 		rc = ext_hw_queue_init(hdev, q);  in hw_queue_init()
523 		rc = int_hw_queue_init(hdev, q);  in hw_queue_init()
527 		rc = cpu_hw_queue_init(hdev, q);  in hw_queue_init()
531 		q->valid = 0;  in hw_queue_init()
536 			q->queue_type);  in hw_queue_init()
544 	q->valid = 1;  in hw_queue_init()
557 static void hw_queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)  in hw_queue_fini()  argument
559 	if (!q->valid)  in hw_queue_fini()
580 	if (q->queue_type == QUEUE_TYPE_INT)  in hw_queue_fini()
583 	kfree(q->shadow_queue);  in hw_queue_fini()
585 	if (q->queue_type == QUEUE_TYPE_CPU)  in hw_queue_fini()
588 					(void *) (uintptr_t) q->kernel_address);  in hw_queue_fini()
592 					(void *) (uintptr_t) q->kernel_address,  in hw_queue_fini()
593 					q->bus_address);  in hw_queue_fini()
599 	struct hl_hw_queue *q;  in hl_hw_queues_create()  local
611 	for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;  in hl_hw_queues_create()
612 			i < HL_MAX_QUEUES ; i++, q_ready_cnt++, q++) {  in hl_hw_queues_create()
614 		q->queue_type = asic->hw_queues_props[i].type;  in hl_hw_queues_create()
615 		rc = hw_queue_init(hdev, q, i);  in hl_hw_queues_create()
626 	for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)  in hl_hw_queues_create()
627 		hw_queue_fini(hdev, q);  in hl_hw_queues_create()
636 	struct hl_hw_queue *q;  in hl_hw_queues_destroy()  local
639 	for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++)  in hl_hw_queues_destroy()
640 		hw_queue_fini(hdev, q);  in hl_hw_queues_destroy()
647 	struct hl_hw_queue *q;  in hl_hw_queue_reset()  local
650 	for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++) {  in hl_hw_queue_reset()
651 		if ((!q->valid) ||  in hl_hw_queue_reset()
652 			((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))  in hl_hw_queue_reset()
654 		q->pi = q->ci = 0;  in hl_hw_queue_reset()