19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)  in hl_device_disabled_or_in_reset()  argument
21 	if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))  in hl_device_disabled_or_in_reset()
27 enum hl_device_status hl_device_status(struct hl_device *hdev)  in hl_device_status()  argument
31 	if (hdev->disabled)  in hl_device_status()
33 	else if (atomic_read(&hdev->in_reset))  in hl_device_status()
44 	struct hl_device *hdev;  in hpriv_release()  local
48 	hdev = hpriv->hdev;  in hpriv_release()
56 	mutex_lock(&hdev->fpriv_list_lock);  in hpriv_release()
58 	hdev->compute_ctx = NULL;  in hpriv_release()
59 	mutex_unlock(&hdev->fpriv_list_lock);  in hpriv_release()
86 	hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);  in hl_device_release()
87 	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);  in hl_device_release()
99 	struct hl_device *hdev;  in hl_device_release_ctrl()  local
103 	hdev = hpriv->hdev;  in hl_device_release_ctrl()
105 	mutex_lock(&hdev->fpriv_list_lock);  in hl_device_release_ctrl()
107 	mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_release_ctrl()
174 static int device_init_cdev(struct hl_device *hdev, struct class *hclass,  in device_init_cdev()  argument
187 	(*dev)->devt = MKDEV(hdev->major, minor);  in device_init_cdev()
190 	dev_set_drvdata(*dev, hdev);  in device_init_cdev()
196 static int device_cdev_sysfs_add(struct hl_device *hdev)  in device_cdev_sysfs_add()  argument
200 	rc = cdev_device_add(&hdev->cdev, hdev->dev);  in device_cdev_sysfs_add()
202 		dev_err(hdev->dev,  in device_cdev_sysfs_add()
207 	rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);  in device_cdev_sysfs_add()
209 		dev_err(hdev->dev,  in device_cdev_sysfs_add()
215 	rc = hl_sysfs_init(hdev);  in device_cdev_sysfs_add()
217 		dev_err(hdev->dev, "failed to initialize sysfs\n");  in device_cdev_sysfs_add()
221 	hdev->cdev_sysfs_created = true;  in device_cdev_sysfs_add()
226 	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);  in device_cdev_sysfs_add()
228 	cdev_device_del(&hdev->cdev, hdev->dev);  in device_cdev_sysfs_add()
232 static void device_cdev_sysfs_del(struct hl_device *hdev)  in device_cdev_sysfs_del()  argument
234 	if (!hdev->cdev_sysfs_created)  in device_cdev_sysfs_del()
237 	hl_sysfs_fini(hdev);  in device_cdev_sysfs_del()
238 	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);  in device_cdev_sysfs_del()
239 	cdev_device_del(&hdev->cdev, hdev->dev);  in device_cdev_sysfs_del()
242 	put_device(hdev->dev);  in device_cdev_sysfs_del()
243 	put_device(hdev->dev_ctrl);  in device_cdev_sysfs_del()
254 static int device_early_init(struct hl_device *hdev)  in device_early_init()  argument
259 	switch (hdev->asic_type) {  in device_early_init()
261 		goya_set_asic_funcs(hdev);  in device_early_init()
262 		strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));  in device_early_init()
265 		gaudi_set_asic_funcs(hdev);  in device_early_init()
266 		sprintf(hdev->asic_name, "GAUDI");  in device_early_init()
269 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",  in device_early_init()
270 			hdev->asic_type);  in device_early_init()
274 	rc = hdev->asic_funcs->early_init(hdev);  in device_early_init()
278 	rc = hl_asid_init(hdev);  in device_early_init()
282 	if (hdev->asic_prop.completion_queues_count) {  in device_early_init()
283 		hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,  in device_early_init()
284 				sizeof(*hdev->cq_wq),  in device_early_init()
286 		if (!hdev->cq_wq) {  in device_early_init()
292 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {  in device_early_init()
294 		hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);  in device_early_init()
295 		if (hdev->cq_wq[i] == NULL) {  in device_early_init()
296 			dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");  in device_early_init()
302 	hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);  in device_early_init()
303 	if (hdev->eq_wq == NULL) {  in device_early_init()
304 		dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");  in device_early_init()
309 	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),  in device_early_init()
311 	if (!hdev->hl_chip_info) {  in device_early_init()
316 	hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE,  in device_early_init()
319 	if (!hdev->idle_busy_ts_arr) {  in device_early_init()
324 	rc = hl_mmu_if_set_funcs(hdev);  in device_early_init()
328 	hl_cb_mgr_init(&hdev->kernel_cb_mgr);  in device_early_init()
330 	mutex_init(&hdev->send_cpu_message_lock);  in device_early_init()
331 	mutex_init(&hdev->debug_lock);  in device_early_init()
332 	mutex_init(&hdev->mmu_cache_lock);  in device_early_init()
333 	INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);  in device_early_init()
334 	spin_lock_init(&hdev->hw_queues_mirror_lock);  in device_early_init()
335 	INIT_LIST_HEAD(&hdev->fpriv_list);  in device_early_init()
336 	mutex_init(&hdev->fpriv_list_lock);  in device_early_init()
337 	atomic_set(&hdev->in_reset, 0);  in device_early_init()
342 	kfree(hdev->idle_busy_ts_arr);  in device_early_init()
344 	kfree(hdev->hl_chip_info);  in device_early_init()
346 	destroy_workqueue(hdev->eq_wq);  in device_early_init()
348 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in device_early_init()
349 		if (hdev->cq_wq[i])  in device_early_init()
350 			destroy_workqueue(hdev->cq_wq[i]);  in device_early_init()
351 	kfree(hdev->cq_wq);  in device_early_init()
353 	hl_asid_fini(hdev);  in device_early_init()
355 	if (hdev->asic_funcs->early_fini)  in device_early_init()
356 		hdev->asic_funcs->early_fini(hdev);  in device_early_init()
367 static void device_early_fini(struct hl_device *hdev)  in device_early_fini()  argument
371 	mutex_destroy(&hdev->mmu_cache_lock);  in device_early_fini()
372 	mutex_destroy(&hdev->debug_lock);  in device_early_fini()
373 	mutex_destroy(&hdev->send_cpu_message_lock);  in device_early_fini()
375 	mutex_destroy(&hdev->fpriv_list_lock);  in device_early_fini()
377 	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);  in device_early_fini()
379 	kfree(hdev->idle_busy_ts_arr);  in device_early_fini()
380 	kfree(hdev->hl_chip_info);  in device_early_fini()
382 	destroy_workqueue(hdev->eq_wq);  in device_early_fini()
384 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in device_early_fini()
385 		destroy_workqueue(hdev->cq_wq[i]);  in device_early_fini()
386 	kfree(hdev->cq_wq);  in device_early_fini()
388 	hl_asid_fini(hdev);  in device_early_fini()
390 	if (hdev->asic_funcs->early_fini)  in device_early_fini()
391 		hdev->asic_funcs->early_fini(hdev);  in device_early_fini()
396 	struct hl_device *hdev = container_of(work, struct hl_device,  in set_freq_to_low_job()  local
399 	mutex_lock(&hdev->fpriv_list_lock);  in set_freq_to_low_job()
401 	if (!hdev->compute_ctx)  in set_freq_to_low_job()
402 		hl_device_set_frequency(hdev, PLL_LOW);  in set_freq_to_low_job()
404 	mutex_unlock(&hdev->fpriv_list_lock);  in set_freq_to_low_job()
406 	schedule_delayed_work(&hdev->work_freq,  in set_freq_to_low_job()
412 	struct hl_device *hdev = container_of(work, struct hl_device,  in hl_device_heartbeat()  local
415 	if (hl_device_disabled_or_in_reset(hdev))  in hl_device_heartbeat()
418 	if (!hdev->asic_funcs->send_heartbeat(hdev))  in hl_device_heartbeat()
421 	dev_err(hdev->dev, "Device heartbeat failed!\n");  in hl_device_heartbeat()
422 	hl_device_reset(hdev, true, false);  in hl_device_heartbeat()
427 	schedule_delayed_work(&hdev->work_heartbeat,  in hl_device_heartbeat()
439 static int device_late_init(struct hl_device *hdev)  in device_late_init()  argument
443 	if (hdev->asic_funcs->late_init) {  in device_late_init()
444 		rc = hdev->asic_funcs->late_init(hdev);  in device_late_init()
446 			dev_err(hdev->dev,  in device_late_init()
452 	hdev->high_pll = hdev->asic_prop.high_pll;  in device_late_init()
455 	hdev->curr_pll_profile = PLL_LOW;  in device_late_init()
457 	if (hdev->pm_mng_profile == PM_AUTO)  in device_late_init()
458 		hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);  in device_late_init()
460 		hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);  in device_late_init()
462 	INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);  in device_late_init()
463 	schedule_delayed_work(&hdev->work_freq,  in device_late_init()
466 	if (hdev->heartbeat) {  in device_late_init()
467 		INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);  in device_late_init()
468 		schedule_delayed_work(&hdev->work_heartbeat,  in device_late_init()
472 	hdev->late_init_done = true;  in device_late_init()
483 static void device_late_fini(struct hl_device *hdev)  in device_late_fini()  argument
485 	if (!hdev->late_init_done)  in device_late_fini()
488 	cancel_delayed_work_sync(&hdev->work_freq);  in device_late_fini()
489 	if (hdev->heartbeat)  in device_late_fini()
490 		cancel_delayed_work_sync(&hdev->work_heartbeat);  in device_late_fini()
492 	if (hdev->asic_funcs->late_fini)  in device_late_fini()
493 		hdev->asic_funcs->late_fini(hdev);  in device_late_fini()
495 	hdev->late_init_done = false;  in device_late_fini()
498 uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms)  in hl_device_utilization()  argument
502 	u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx;  in hl_device_utilization()
508 	ts = &hdev->idle_busy_ts_arr[last_index];  in hl_device_utilization()
519 		ts = &hdev->idle_busy_ts_arr[last_index];  in hl_device_utilization()
583 		ts = &hdev->idle_busy_ts_arr[last_index];  in hl_device_utilization()
607 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)  in hl_device_set_frequency()  argument
609 	if ((hdev->pm_mng_profile == PM_MANUAL) ||  in hl_device_set_frequency()
610 			(hdev->curr_pll_profile == freq))  in hl_device_set_frequency()
613 	dev_dbg(hdev->dev, "Changing device frequency to %s\n",  in hl_device_set_frequency()
616 	hdev->asic_funcs->set_pll_profile(hdev, freq);  in hl_device_set_frequency()
618 	hdev->curr_pll_profile = freq;  in hl_device_set_frequency()
623 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)  in hl_device_set_debug_mode()  argument
627 	mutex_lock(&hdev->debug_lock);  in hl_device_set_debug_mode()
630 		if (!hdev->in_debug) {  in hl_device_set_debug_mode()
631 			dev_err(hdev->dev,  in hl_device_set_debug_mode()
637 		if (!hdev->hard_reset_pending)  in hl_device_set_debug_mode()
638 			hdev->asic_funcs->halt_coresight(hdev);  in hl_device_set_debug_mode()
640 		hdev->in_debug = 0;  in hl_device_set_debug_mode()
642 		if (!hdev->hard_reset_pending)  in hl_device_set_debug_mode()
643 			hdev->asic_funcs->set_clock_gating(hdev);  in hl_device_set_debug_mode()
648 	if (hdev->in_debug) {  in hl_device_set_debug_mode()
649 		dev_err(hdev->dev,  in hl_device_set_debug_mode()
655 	hdev->asic_funcs->disable_clock_gating(hdev);  in hl_device_set_debug_mode()
656 	hdev->in_debug = 1;  in hl_device_set_debug_mode()
659 	mutex_unlock(&hdev->debug_lock);  in hl_device_set_debug_mode()
673 int hl_device_suspend(struct hl_device *hdev)  in hl_device_suspend()  argument
677 	pci_save_state(hdev->pdev);  in hl_device_suspend()
680 	rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_suspend()
682 		dev_err(hdev->dev, "Can't suspend while in reset\n");  in hl_device_suspend()
687 	hdev->disabled = true;  in hl_device_suspend()
693 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_device_suspend()
694 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_device_suspend()
697 	mutex_lock(&hdev->send_cpu_message_lock);  in hl_device_suspend()
698 	mutex_unlock(&hdev->send_cpu_message_lock);  in hl_device_suspend()
700 	rc = hdev->asic_funcs->suspend(hdev);  in hl_device_suspend()
702 		dev_err(hdev->dev,  in hl_device_suspend()
706 	pci_disable_device(hdev->pdev);  in hl_device_suspend()
707 	pci_set_power_state(hdev->pdev, PCI_D3hot);  in hl_device_suspend()
721 int hl_device_resume(struct hl_device *hdev)  in hl_device_resume()  argument
725 	pci_set_power_state(hdev->pdev, PCI_D0);  in hl_device_resume()
726 	pci_restore_state(hdev->pdev);  in hl_device_resume()
727 	rc = pci_enable_device_mem(hdev->pdev);  in hl_device_resume()
729 		dev_err(hdev->dev,  in hl_device_resume()
734 	pci_set_master(hdev->pdev);  in hl_device_resume()
736 	rc = hdev->asic_funcs->resume(hdev);  in hl_device_resume()
738 		dev_err(hdev->dev, "Failed to resume device after suspend\n");  in hl_device_resume()
743 	hdev->disabled = false;  in hl_device_resume()
744 	atomic_set(&hdev->in_reset, 0);  in hl_device_resume()
746 	rc = hl_device_reset(hdev, true, false);  in hl_device_resume()
748 		dev_err(hdev->dev, "Failed to reset device during resume\n");  in hl_device_resume()
755 	pci_clear_master(hdev->pdev);  in hl_device_resume()
756 	pci_disable_device(hdev->pdev);  in hl_device_resume()
761 static int device_kill_open_processes(struct hl_device *hdev)  in device_kill_open_processes()  argument
767 	if (hdev->pldm)  in device_kill_open_processes()
775 	if (!list_empty(&hdev->fpriv_list))  in device_kill_open_processes()
778 	mutex_lock(&hdev->fpriv_list_lock);  in device_kill_open_processes()
783 	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {  in device_kill_open_processes()
786 			dev_info(hdev->dev, "Killing user process pid=%d\n",  in device_kill_open_processes()
795 	mutex_unlock(&hdev->fpriv_list_lock);  in device_kill_open_processes()
805 	while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {  in device_kill_open_processes()
806 		dev_info(hdev->dev,  in device_kill_open_processes()
814 	return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;  in device_kill_open_processes()
821 	struct hl_device *hdev = device_reset_work->hdev;  in device_hard_reset_pending()  local
823 	hl_device_reset(hdev, true, true);  in device_hard_reset_pending()
846 int hl_device_reset(struct hl_device *hdev, bool hard_reset,  in hl_device_reset()  argument
851 	if (!hdev->init_done) {  in hl_device_reset()
852 		dev_err(hdev->dev,  in hl_device_reset()
857 	if ((!hard_reset) && (!hdev->supports_soft_reset)) {  in hl_device_reset()
858 		dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");  in hl_device_reset()
869 		rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_reset()
883 			if (hl_fw_send_pci_access_msg(hdev,  in hl_device_reset()
885 				dev_warn(hdev->dev,  in hl_device_reset()
890 		hdev->disabled = true;  in hl_device_reset()
895 		hdev->asic_funcs->hw_queues_lock(hdev);  in hl_device_reset()
896 		hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_device_reset()
899 		mutex_lock(&hdev->fpriv_list_lock);  in hl_device_reset()
900 		mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_reset()
902 		dev_err(hdev->dev, "Going to RESET device!\n");  in hl_device_reset()
909 		hdev->hard_reset_pending = true;  in hl_device_reset()
925 		device_reset_work->hdev = hdev;  in hl_device_reset()
932 		device_late_fini(hdev);  in hl_device_reset()
938 		mutex_lock(&hdev->send_cpu_message_lock);  in hl_device_reset()
939 		mutex_unlock(&hdev->send_cpu_message_lock);  in hl_device_reset()
947 	hdev->asic_funcs->halt_engines(hdev, hard_reset);  in hl_device_reset()
950 	hl_cs_rollback_all(hdev);  in hl_device_reset()
957 		rc = device_kill_open_processes(hdev);  in hl_device_reset()
959 			dev_crit(hdev->dev,  in hl_device_reset()
967 		flush_workqueue(hdev->eq_wq);  in hl_device_reset()
971 	hdev->asic_funcs->hw_fini(hdev, hard_reset);  in hl_device_reset()
975 		if (hl_ctx_put(hdev->kernel_ctx) == 1)  in hl_device_reset()
976 			hdev->kernel_ctx = NULL;  in hl_device_reset()
977 		hl_vm_fini(hdev);  in hl_device_reset()
978 		hl_mmu_fini(hdev);  in hl_device_reset()
979 		hl_eq_reset(hdev, &hdev->event_queue);  in hl_device_reset()
983 	hl_hw_queue_reset(hdev, hard_reset);  in hl_device_reset()
984 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in hl_device_reset()
985 		hl_cq_reset(hdev, &hdev->completion_queue[i]);  in hl_device_reset()
987 	hdev->idle_busy_ts_idx = 0;  in hl_device_reset()
988 	hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0);  in hl_device_reset()
989 	hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0);  in hl_device_reset()
991 	if (hdev->cs_active_cnt)  in hl_device_reset()
992 		dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n",  in hl_device_reset()
993 			hdev->cs_active_cnt);  in hl_device_reset()
995 	mutex_lock(&hdev->fpriv_list_lock);  in hl_device_reset()
998 	if (hdev->compute_ctx) {  in hl_device_reset()
999 		atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);  in hl_device_reset()
1000 		hdev->compute_ctx->thread_ctx_switch_wait_token = 0;  in hl_device_reset()
1003 	mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_reset()
1008 		hdev->device_cpu_disabled = false;  in hl_device_reset()
1009 		hdev->hard_reset_pending = false;  in hl_device_reset()
1011 		if (hdev->kernel_ctx) {  in hl_device_reset()
1012 			dev_crit(hdev->dev,  in hl_device_reset()
1018 		rc = hl_mmu_init(hdev);  in hl_device_reset()
1020 			dev_err(hdev->dev,  in hl_device_reset()
1026 		hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),  in hl_device_reset()
1028 		if (!hdev->kernel_ctx) {  in hl_device_reset()
1033 		hdev->compute_ctx = NULL;  in hl_device_reset()
1035 		rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);  in hl_device_reset()
1037 			dev_err(hdev->dev,  in hl_device_reset()
1039 			kfree(hdev->kernel_ctx);  in hl_device_reset()
1040 			hdev->kernel_ctx = NULL;  in hl_device_reset()
1049 	hdev->disabled = false;  in hl_device_reset()
1051 	rc = hdev->asic_funcs->hw_init(hdev);  in hl_device_reset()
1053 		dev_err(hdev->dev,  in hl_device_reset()
1059 	rc = hdev->asic_funcs->test_queues(hdev);  in hl_device_reset()
1061 		dev_err(hdev->dev,  in hl_device_reset()
1067 		rc = device_late_init(hdev);  in hl_device_reset()
1069 			dev_err(hdev->dev,  in hl_device_reset()
1074 		rc = hl_vm_init(hdev);  in hl_device_reset()
1076 			dev_err(hdev->dev,  in hl_device_reset()
1081 		hl_set_max_power(hdev);  in hl_device_reset()
1083 		rc = hdev->asic_funcs->soft_reset_late_init(hdev);  in hl_device_reset()
1085 			dev_err(hdev->dev,  in hl_device_reset()
1091 	atomic_set(&hdev->in_reset, 0);  in hl_device_reset()
1094 		hdev->hard_reset_cnt++;  in hl_device_reset()
1096 		hdev->soft_reset_cnt++;  in hl_device_reset()
1098 	dev_warn(hdev->dev, "Successfully finished resetting the device\n");  in hl_device_reset()
1103 	hdev->disabled = true;  in hl_device_reset()
1106 		dev_err(hdev->dev,  in hl_device_reset()
1108 		hdev->hard_reset_cnt++;  in hl_device_reset()
1110 		dev_err(hdev->dev,  in hl_device_reset()
1112 		hdev->soft_reset_cnt++;  in hl_device_reset()
1117 	atomic_set(&hdev->in_reset, 0);  in hl_device_reset()
1131 int hl_device_init(struct hl_device *hdev, struct class *hclass)  in hl_device_init()  argument
1137 	name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);  in hl_device_init()
1144 	rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,  in hl_device_init()
1145 				&hdev->cdev, &hdev->dev);  in hl_device_init()
1152 	name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);  in hl_device_init()
1159 	rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,  in hl_device_init()
1160 				name, &hdev->cdev_ctrl, &hdev->dev_ctrl);  in hl_device_init()
1168 	rc = device_early_init(hdev);  in hl_device_init()
1176 	rc = hdev->asic_funcs->sw_init(hdev);  in hl_device_init()
1185 	rc = hl_hw_queues_create(hdev);  in hl_device_init()
1187 		dev_err(hdev->dev, "failed to initialize kernel queues\n");  in hl_device_init()
1191 	cq_cnt = hdev->asic_prop.completion_queues_count;  in hl_device_init()
1199 		hdev->completion_queue = kcalloc(cq_cnt,  in hl_device_init()
1200 				sizeof(*hdev->completion_queue),  in hl_device_init()
1203 		if (!hdev->completion_queue) {  in hl_device_init()
1204 			dev_err(hdev->dev,  in hl_device_init()
1212 		rc = hl_cq_init(hdev, &hdev->completion_queue[i],  in hl_device_init()
1213 				hdev->asic_funcs->get_queue_id_for_cq(hdev, i));  in hl_device_init()
1215 			dev_err(hdev->dev,  in hl_device_init()
1219 		hdev->completion_queue[i].cq_idx = i;  in hl_device_init()
1227 	rc = hl_eq_init(hdev, &hdev->event_queue);  in hl_device_init()
1229 		dev_err(hdev->dev, "failed to initialize event queue\n");  in hl_device_init()
1234 	rc = hl_mmu_init(hdev);  in hl_device_init()
1236 		dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");  in hl_device_init()
1241 	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);  in hl_device_init()
1242 	if (!hdev->kernel_ctx) {  in hl_device_init()
1247 	hdev->compute_ctx = NULL;  in hl_device_init()
1249 	rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);  in hl_device_init()
1251 		dev_err(hdev->dev, "failed to initialize kernel context\n");  in hl_device_init()
1252 		kfree(hdev->kernel_ctx);  in hl_device_init()
1256 	rc = hl_cb_pool_init(hdev);  in hl_device_init()
1258 		dev_err(hdev->dev, "failed to initialize CB pool\n");  in hl_device_init()
1262 	hl_debugfs_add_device(hdev);  in hl_device_init()
1264 	if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {  in hl_device_init()
1265 		dev_info(hdev->dev,  in hl_device_init()
1267 		hdev->asic_funcs->halt_engines(hdev, true);  in hl_device_init()
1268 		hdev->asic_funcs->hw_fini(hdev, true);  in hl_device_init()
1281 	hdev->disabled = false;  in hl_device_init()
1283 	rc = hdev->asic_funcs->hw_init(hdev);  in hl_device_init()
1285 		dev_err(hdev->dev, "failed to initialize the H/W\n");  in hl_device_init()
1291 	rc = hdev->asic_funcs->test_queues(hdev);  in hl_device_init()
1293 		dev_err(hdev->dev, "Failed to detect if device is alive\n");  in hl_device_init()
1298 	rc = device_late_init(hdev);  in hl_device_init()
1300 		dev_err(hdev->dev, "Failed late initialization\n");  in hl_device_init()
1305 	dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",  in hl_device_init()
1306 		hdev->asic_name,  in hl_device_init()
1307 		hdev->asic_prop.dram_size / 1024 / 1024 / 1024);  in hl_device_init()
1309 	rc = hl_vm_init(hdev);  in hl_device_init()
1311 		dev_err(hdev->dev, "Failed to initialize memory module\n");  in hl_device_init()
1322 	rc = device_cdev_sysfs_add(hdev);  in hl_device_init()
1324 		dev_err(hdev->dev,  in hl_device_init()
1333 	hl_set_max_power(hdev);  in hl_device_init()
1341 	rc = hl_hwmon_init(hdev);  in hl_device_init()
1343 		dev_err(hdev->dev, "Failed to initialize hwmon\n");  in hl_device_init()
1348 	dev_notice(hdev->dev,  in hl_device_init()
1351 	hdev->init_done = true;  in hl_device_init()
1356 	if (hl_ctx_put(hdev->kernel_ctx) != 1)  in hl_device_init()
1357 		dev_err(hdev->dev,  in hl_device_init()
1360 	hl_mmu_fini(hdev);  in hl_device_init()
1362 	hl_eq_fini(hdev, &hdev->event_queue);  in hl_device_init()
1365 		hl_cq_fini(hdev, &hdev->completion_queue[i]);  in hl_device_init()
1366 	kfree(hdev->completion_queue);  in hl_device_init()
1368 	hl_hw_queues_destroy(hdev);  in hl_device_init()
1370 	hdev->asic_funcs->sw_fini(hdev);  in hl_device_init()
1372 	device_early_fini(hdev);  in hl_device_init()
1374 	put_device(hdev->dev_ctrl);  in hl_device_init()
1376 	put_device(hdev->dev);  in hl_device_init()
1378 	hdev->disabled = true;  in hl_device_init()
1380 		device_cdev_sysfs_add(hdev);  in hl_device_init()
1381 	if (hdev->pdev)  in hl_device_init()
1382 		dev_err(&hdev->pdev->dev,  in hl_device_init()
1384 			hdev->id / 2);  in hl_device_init()
1387 			hdev->id / 2);  in hl_device_init()
1399 void hl_device_fini(struct hl_device *hdev)  in hl_device_fini()  argument
1404 	dev_info(hdev->dev, "Removing device\n");  in hl_device_fini()
1416 	rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_fini()
1419 		rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_fini()
1427 	hdev->disabled = true;  in hl_device_fini()
1432 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_device_fini()
1433 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_device_fini()
1436 	mutex_lock(&hdev->fpriv_list_lock);  in hl_device_fini()
1437 	mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_fini()
1439 	hdev->hard_reset_pending = true;  in hl_device_fini()
1441 	hl_hwmon_fini(hdev);  in hl_device_fini()
1443 	device_late_fini(hdev);  in hl_device_fini()
1445 	hl_debugfs_remove_device(hdev);  in hl_device_fini()
1452 	hdev->asic_funcs->halt_engines(hdev, true);  in hl_device_fini()
1455 	hl_cs_rollback_all(hdev);  in hl_device_fini()
1461 	rc = device_kill_open_processes(hdev);  in hl_device_fini()
1463 		dev_crit(hdev->dev, "Failed to kill all open processes\n");  in hl_device_fini()
1465 	hl_cb_pool_fini(hdev);  in hl_device_fini()
1468 	hdev->asic_funcs->hw_fini(hdev, true);  in hl_device_fini()
1471 	if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))  in hl_device_fini()
1472 		dev_err(hdev->dev, "kernel ctx is still alive\n");  in hl_device_fini()
1474 	hl_vm_fini(hdev);  in hl_device_fini()
1476 	hl_mmu_fini(hdev);  in hl_device_fini()
1478 	hl_eq_fini(hdev, &hdev->event_queue);  in hl_device_fini()
1480 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in hl_device_fini()
1481 		hl_cq_fini(hdev, &hdev->completion_queue[i]);  in hl_device_fini()
1482 	kfree(hdev->completion_queue);  in hl_device_fini()
1484 	hl_hw_queues_destroy(hdev);  in hl_device_fini()
1487 	hdev->asic_funcs->sw_fini(hdev);  in hl_device_fini()
1489 	device_early_fini(hdev);  in hl_device_fini()
1492 	device_cdev_sysfs_del(hdev);  in hl_device_fini()
1510 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)  in hl_rreg()  argument
1512 	return readl(hdev->rmmio + reg);  in hl_rreg()
1525 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)  in hl_wreg()  argument
1527 	writel(val, hdev->rmmio + reg);  in hl_wreg()