Lines Matching +full:no +full:- +full:reset +full:- +full:on +full:- +full:init
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
20 if (atomic_read(&hdev->in_reset)) in hl_device_status()
22 else if (hdev->needs_reset) in hl_device_status()
24 else if (hdev->disabled) in hl_device_status()
26 else if (!hdev->init_done) in hl_device_status()
64 hdev = hpriv->hdev; in hpriv_release()
66 put_pid(hpriv->taskpid); in hpriv_release()
70 mutex_destroy(&hpriv->restore_phase_mutex); in hpriv_release()
72 mutex_lock(&hdev->fpriv_list_lock); in hpriv_release()
73 list_del(&hpriv->dev_node); in hpriv_release()
74 hdev->compute_ctx = NULL; in hpriv_release()
75 mutex_unlock(&hdev->fpriv_list_lock); in hpriv_release()
79 if ((!hdev->pldm) && (hdev->pdev) && in hpriv_release()
80 (!hdev->asic_funcs->is_device_idle(hdev, in hpriv_release()
83 dev_err(hdev->dev, in hpriv_release()
90 if ((hdev->reset_if_device_not_idle && !device_is_idle) in hpriv_release()
91 || hdev->reset_upon_device_release) in hpriv_release()
97 kref_get(&hpriv->refcount); in hl_hpriv_get()
102 return kref_put(&hpriv->refcount, hpriv_release); in hl_hpriv_put()
106 * hl_device_release - release function for habanalabs device
115 struct hl_fpriv *hpriv = filp->private_data; in hl_device_release()
116 struct hl_device *hdev = hpriv->hdev; in hl_device_release()
118 filp->private_data = NULL; in hl_device_release()
122 put_pid(hpriv->taskpid); in hl_device_release()
129 hl_release_pending_user_interrupts(hpriv->hdev); in hl_device_release()
131 hl_cb_mgr_fini(hdev, &hpriv->cb_mgr); in hl_device_release()
132 hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr); in hl_device_release()
135 dev_notice(hdev->dev, in hl_device_release()
138 hdev->last_open_session_duration_jif = in hl_device_release()
139 jiffies - hdev->last_successful_open_jif; in hl_device_release()
146 struct hl_fpriv *hpriv = filp->private_data; in hl_device_release_ctrl()
147 struct hl_device *hdev = hpriv->hdev; in hl_device_release_ctrl()
149 filp->private_data = NULL; in hl_device_release_ctrl()
156 mutex_lock(&hdev->fpriv_list_lock); in hl_device_release_ctrl()
157 list_del(&hpriv->dev_node); in hl_device_release_ctrl()
158 mutex_unlock(&hdev->fpriv_list_lock); in hl_device_release_ctrl()
160 put_pid(hpriv->taskpid); in hl_device_release_ctrl()
168 * hl_mmap - mmap function for habanalabs device
173 * Called when process does an mmap on habanalabs device. Call the device's mmap
178 struct hl_fpriv *hpriv = filp->private_data; in hl_mmap()
179 struct hl_device *hdev = hpriv->hdev; in hl_mmap()
184 return -ENODEV; in hl_mmap()
187 vm_pgoff = vma->vm_pgoff; in hl_mmap()
188 vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff); in hl_mmap()
198 return -EINVAL; in hl_mmap()
224 * device_init_cdev - Initialize cdev and device for habanalabs device
242 cdev->owner = THIS_MODULE; in device_init_cdev()
246 return -ENOMEM; in device_init_cdev()
249 (*dev)->devt = MKDEV(hdev->major, minor); in device_init_cdev()
250 (*dev)->class = hclass; in device_init_cdev()
251 (*dev)->release = device_release_func; in device_init_cdev()
262 rc = cdev_device_add(&hdev->cdev, hdev->dev); in device_cdev_sysfs_add()
264 dev_err(hdev->dev, in device_cdev_sysfs_add()
269 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl); in device_cdev_sysfs_add()
271 dev_err(hdev->dev, in device_cdev_sysfs_add()
279 dev_err(hdev->dev, "failed to initialize sysfs\n"); in device_cdev_sysfs_add()
283 hdev->cdev_sysfs_created = true; in device_cdev_sysfs_add()
288 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); in device_cdev_sysfs_add()
290 cdev_device_del(&hdev->cdev, hdev->dev); in device_cdev_sysfs_add()
296 if (!hdev->cdev_sysfs_created) in device_cdev_sysfs_del()
300 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); in device_cdev_sysfs_del()
301 cdev_device_del(&hdev->cdev, hdev->dev); in device_cdev_sysfs_del()
304 put_device(hdev->dev); in device_cdev_sysfs_del()
305 put_device(hdev->dev_ctrl); in device_cdev_sysfs_del()
313 struct hl_device *hdev = device_reset_work->hdev; in device_hard_reset_pending()
319 if (device_reset_work->fw_reset) in device_hard_reset_pending()
323 if ((rc == -EBUSY) && !hdev->device_fini_pending) { in device_hard_reset_pending()
324 dev_info(hdev->dev, in device_hard_reset_pending()
325 "Could not reset device. will try again in %u seconds", in device_hard_reset_pending()
328 queue_delayed_work(device_reset_work->wq, in device_hard_reset_pending()
329 &device_reset_work->reset_work, in device_hard_reset_pending()
335 * device_early_init - do some early initialization for the habanalabs device
347 switch (hdev->asic_type) { in device_early_init()
350 strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name)); in device_early_init()
354 strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name)); in device_early_init()
358 strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name)); in device_early_init()
361 dev_err(hdev->dev, "Unrecognized ASIC type %d\n", in device_early_init()
362 hdev->asic_type); in device_early_init()
363 return -EINVAL; in device_early_init()
366 rc = hdev->asic_funcs->early_init(hdev); in device_early_init()
374 if (hdev->asic_prop.completion_queues_count) { in device_early_init()
375 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count, in device_early_init()
376 sizeof(*hdev->cq_wq), in device_early_init()
378 if (!hdev->cq_wq) { in device_early_init()
379 rc = -ENOMEM; in device_early_init()
384 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in device_early_init()
385 snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i); in device_early_init()
386 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name); in device_early_init()
387 if (hdev->cq_wq[i] == NULL) { in device_early_init()
388 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n"); in device_early_init()
389 rc = -ENOMEM; in device_early_init()
394 hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0); in device_early_init()
395 if (hdev->eq_wq == NULL) { in device_early_init()
396 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n"); in device_early_init()
397 rc = -ENOMEM; in device_early_init()
401 hdev->sob_reset_wq = alloc_workqueue("hl-sob-reset", WQ_UNBOUND, 0); in device_early_init()
402 if (!hdev->sob_reset_wq) { in device_early_init()
403 dev_err(hdev->dev, in device_early_init()
404 "Failed to allocate SOB reset workqueue\n"); in device_early_init()
405 rc = -ENOMEM; in device_early_init()
409 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), in device_early_init()
411 if (!hdev->hl_chip_info) { in device_early_init()
412 rc = -ENOMEM; in device_early_init()
420 hl_cb_mgr_init(&hdev->kernel_cb_mgr); in device_early_init()
422 hdev->device_reset_work.wq = in device_early_init()
424 if (!hdev->device_reset_work.wq) { in device_early_init()
425 rc = -ENOMEM; in device_early_init()
426 dev_err(hdev->dev, "Failed to create device reset WQ\n"); in device_early_init()
430 INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, in device_early_init()
432 hdev->device_reset_work.hdev = hdev; in device_early_init()
433 hdev->device_fini_pending = 0; in device_early_init()
435 mutex_init(&hdev->send_cpu_message_lock); in device_early_init()
436 mutex_init(&hdev->debug_lock); in device_early_init()
437 INIT_LIST_HEAD(&hdev->cs_mirror_list); in device_early_init()
438 spin_lock_init(&hdev->cs_mirror_lock); in device_early_init()
439 INIT_LIST_HEAD(&hdev->fpriv_list); in device_early_init()
440 mutex_init(&hdev->fpriv_list_lock); in device_early_init()
441 atomic_set(&hdev->in_reset, 0); in device_early_init()
446 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr); in device_early_init()
448 kfree(hdev->hl_chip_info); in device_early_init()
450 destroy_workqueue(hdev->sob_reset_wq); in device_early_init()
452 destroy_workqueue(hdev->eq_wq); in device_early_init()
454 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in device_early_init()
455 if (hdev->cq_wq[i]) in device_early_init()
456 destroy_workqueue(hdev->cq_wq[i]); in device_early_init()
457 kfree(hdev->cq_wq); in device_early_init()
461 if (hdev->asic_funcs->early_fini) in device_early_init()
462 hdev->asic_funcs->early_fini(hdev); in device_early_init()
468 * device_early_fini - finalize all that was done in device_early_init
477 mutex_destroy(&hdev->debug_lock); in device_early_fini()
478 mutex_destroy(&hdev->send_cpu_message_lock); in device_early_fini()
480 mutex_destroy(&hdev->fpriv_list_lock); in device_early_fini()
482 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr); in device_early_fini()
484 kfree(hdev->hl_chip_info); in device_early_fini()
486 destroy_workqueue(hdev->sob_reset_wq); in device_early_fini()
487 destroy_workqueue(hdev->eq_wq); in device_early_fini()
488 destroy_workqueue(hdev->device_reset_work.wq); in device_early_fini()
490 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in device_early_fini()
491 destroy_workqueue(hdev->cq_wq[i]); in device_early_fini()
492 kfree(hdev->cq_wq); in device_early_fini()
496 if (hdev->asic_funcs->early_fini) in device_early_fini()
497 hdev->asic_funcs->early_fini(hdev); in device_early_fini()
505 mutex_lock(&hdev->fpriv_list_lock); in set_freq_to_low_job()
507 if (!hdev->compute_ctx) in set_freq_to_low_job()
510 mutex_unlock(&hdev->fpriv_list_lock); in set_freq_to_low_job()
512 schedule_delayed_work(&hdev->work_freq, in set_freq_to_low_job()
524 if (!hdev->asic_funcs->send_heartbeat(hdev)) in hl_device_heartbeat()
527 dev_err(hdev->dev, "Device heartbeat failed!\n"); in hl_device_heartbeat()
533 schedule_delayed_work(&hdev->work_heartbeat, in hl_device_heartbeat()
538 * device_late_init - do late stuff initialization for the habanalabs device
549 if (hdev->asic_funcs->late_init) { in device_late_init()
550 rc = hdev->asic_funcs->late_init(hdev); in device_late_init()
552 dev_err(hdev->dev, in device_late_init()
558 hdev->high_pll = hdev->asic_prop.high_pll; in device_late_init()
561 hdev->curr_pll_profile = PLL_LOW; in device_late_init()
563 if (hdev->pm_mng_profile == PM_AUTO) in device_late_init()
564 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW); in device_late_init()
566 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST); in device_late_init()
568 INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job); in device_late_init()
569 schedule_delayed_work(&hdev->work_freq, in device_late_init()
572 if (hdev->heartbeat) { in device_late_init()
573 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat); in device_late_init()
574 schedule_delayed_work(&hdev->work_heartbeat, in device_late_init()
578 hdev->late_init_done = true; in device_late_init()
584 * device_late_fini - finalize all that was done in device_late_init
591 if (!hdev->late_init_done) in device_late_fini()
594 cancel_delayed_work_sync(&hdev->work_freq); in device_late_fini()
595 if (hdev->heartbeat) in device_late_fini()
596 cancel_delayed_work_sync(&hdev->work_heartbeat); in device_late_fini()
598 if (hdev->asic_funcs->late_fini) in device_late_fini()
599 hdev->asic_funcs->late_fini(hdev); in device_late_fini()
601 hdev->late_init_done = false; in device_late_fini()
609 max_power = hdev->asic_prop.max_power_default; in hl_device_utilization()
610 dc_power = hdev->asic_prop.dc_power_default; in hl_device_utilization()
618 dividend = (curr_power - dc_power) * 100; in hl_device_utilization()
619 *utilization = (u32) div_u64(dividend, (max_power - dc_power)); in hl_device_utilization()
625 * hl_device_set_frequency - set the frequency of the device
630 * Change the frequency if needed. This function has no protection against
635 * Returns 0 if no change was done, otherwise returns 1
639 if ((hdev->pm_mng_profile == PM_MANUAL) || in hl_device_set_frequency()
640 (hdev->curr_pll_profile == freq)) in hl_device_set_frequency()
643 dev_dbg(hdev->dev, "Changing device frequency to %s\n", in hl_device_set_frequency()
646 hdev->asic_funcs->set_pll_profile(hdev, freq); in hl_device_set_frequency()
648 hdev->curr_pll_profile = freq; in hl_device_set_frequency()
657 mutex_lock(&hdev->debug_lock); in hl_device_set_debug_mode()
660 if (!hdev->in_debug) { in hl_device_set_debug_mode()
661 dev_err(hdev->dev, in hl_device_set_debug_mode()
663 rc = -EFAULT; in hl_device_set_debug_mode()
667 if (!hdev->hard_reset_pending) in hl_device_set_debug_mode()
668 hdev->asic_funcs->halt_coresight(hdev); in hl_device_set_debug_mode()
670 hdev->in_debug = 0; in hl_device_set_debug_mode()
672 if (!hdev->hard_reset_pending) in hl_device_set_debug_mode()
673 hdev->asic_funcs->set_clock_gating(hdev); in hl_device_set_debug_mode()
678 if (hdev->in_debug) { in hl_device_set_debug_mode()
679 dev_err(hdev->dev, in hl_device_set_debug_mode()
681 rc = -EFAULT; in hl_device_set_debug_mode()
685 hdev->asic_funcs->disable_clock_gating(hdev); in hl_device_set_debug_mode()
686 hdev->in_debug = 1; in hl_device_set_debug_mode()
689 mutex_unlock(&hdev->debug_lock); in hl_device_set_debug_mode()
699 hdev->asic_funcs->hw_queues_lock(hdev); in take_release_locks()
700 hdev->asic_funcs->hw_queues_unlock(hdev); in take_release_locks()
703 mutex_lock(&hdev->send_cpu_message_lock); in take_release_locks()
704 mutex_unlock(&hdev->send_cpu_message_lock); in take_release_locks()
707 mutex_lock(&hdev->fpriv_list_lock); in take_release_locks()
708 mutex_unlock(&hdev->fpriv_list_lock); in take_release_locks()
721 hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset); in cleanup_resources()
733 * hl_device_suspend - initiate device suspend
738 * Returns 0 for success or an error on failure.
745 pci_save_state(hdev->pdev); in hl_device_suspend()
748 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); in hl_device_suspend()
750 dev_err(hdev->dev, "Can't suspend while in reset\n"); in hl_device_suspend()
751 return -EIO; in hl_device_suspend()
755 hdev->disabled = true; in hl_device_suspend()
759 rc = hdev->asic_funcs->suspend(hdev); in hl_device_suspend()
761 dev_err(hdev->dev, in hl_device_suspend()
765 pci_disable_device(hdev->pdev); in hl_device_suspend()
766 pci_set_power_state(hdev->pdev, PCI_D3hot); in hl_device_suspend()
772 * hl_device_resume - initiate device resume
777 * Returns 0 for success or an error on failure.
784 pci_set_power_state(hdev->pdev, PCI_D0); in hl_device_resume()
785 pci_restore_state(hdev->pdev); in hl_device_resume()
786 rc = pci_enable_device_mem(hdev->pdev); in hl_device_resume()
788 dev_err(hdev->dev, in hl_device_resume()
793 pci_set_master(hdev->pdev); in hl_device_resume()
795 rc = hdev->asic_funcs->resume(hdev); in hl_device_resume()
797 dev_err(hdev->dev, "Failed to resume device after suspend\n"); in hl_device_resume()
802 hdev->disabled = false; in hl_device_resume()
803 atomic_set(&hdev->in_reset, 0); in hl_device_resume()
807 dev_err(hdev->dev, "Failed to reset device during resume\n"); in hl_device_resume()
814 pci_clear_master(hdev->pdev); in hl_device_resume()
815 pci_disable_device(hdev->pdev); in hl_device_resume()
830 if (!list_empty(&hdev->fpriv_list)) in device_kill_open_processes()
836 if (hdev->process_kill_trial_cnt) { in device_kill_open_processes()
846 mutex_lock(&hdev->fpriv_list_lock); in device_kill_open_processes()
851 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) { in device_kill_open_processes()
852 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID); in device_kill_open_processes()
854 dev_info(hdev->dev, "Killing user process pid=%d\n", in device_kill_open_processes()
861 dev_warn(hdev->dev, in device_kill_open_processes()
862 "Can't get task struct for PID so giving up on killing process\n"); in device_kill_open_processes()
863 mutex_unlock(&hdev->fpriv_list_lock); in device_kill_open_processes()
864 return -ETIME; in device_kill_open_processes()
868 mutex_unlock(&hdev->fpriv_list_lock); in device_kill_open_processes()
876 * continuing with the reset. in device_kill_open_processes()
880 while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) { in device_kill_open_processes()
881 dev_dbg(hdev->dev, in device_kill_open_processes()
882 "Waiting for all unmap operations to finish before hard reset\n"); in device_kill_open_processes()
884 pending_cnt--; in device_kill_open_processes()
890 if (list_empty(&hdev->fpriv_list)) in device_kill_open_processes()
894 if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS) in device_kill_open_processes()
895 return -ETIME; in device_kill_open_processes()
897 hdev->process_kill_trial_cnt++; in device_kill_open_processes()
899 return -EBUSY; in device_kill_open_processes()
906 mutex_lock(&hdev->fpriv_list_lock); in device_disable_open_processes()
907 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) in device_disable_open_processes()
908 hpriv->hdev = NULL; in device_disable_open_processes()
909 mutex_unlock(&hdev->fpriv_list_lock); in device_disable_open_processes()
913 * hl_device_reset - reset the device
916 * @flags: reset flags.
921 * Re-initialize all internal data structures
922 * Call ASIC H/W init, late_init
926 * Returns 0 for success or an error on failure.
934 if (!hdev->init_done) { in hl_device_reset()
935 dev_err(hdev->dev, in hl_device_reset()
936 "Can't reset before initialization is done\n"); in hl_device_reset()
944 if (!hard_reset && !hdev->supports_soft_reset) { in hl_device_reset()
949 if (hdev->reset_upon_device_release && in hl_device_reset()
951 dev_dbg(hdev->dev, in hl_device_reset()
952 "Perform %s-reset upon device release\n", in hl_device_reset()
957 if (!hard_reset && !hdev->allow_external_soft_reset) { in hl_device_reset()
963 dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n"); in hl_device_reset()
966 /* Re-entry of reset thread */ in hl_device_reset()
967 if (from_hard_reset_thread && hdev->process_kill_trial_cnt) in hl_device_reset()
971 * Prevent concurrency in this function - only one reset should be in hl_device_reset()
973 * get from the dedicated hard reset thread in hl_device_reset()
977 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); in hl_device_reset()
982 * 'reset cause' is being updated here, because getting here in hl_device_reset()
988 hdev->curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT; in hl_device_reset()
990 hdev->curr_reset_cause = HL_RESET_CAUSE_TDR; in hl_device_reset()
992 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; in hl_device_reset()
994 /* If reset is due to heartbeat, device CPU is no responsive in in hl_device_reset()
995 * which case no point sending PCI disable message to it. in hl_device_reset()
997 * If F/W is performing the reset, no need to send it a message to disable in hl_device_reset()
1002 * us additional interrupts. We disable MSI/MSI-X at in hl_device_reset()
1012 dev_warn(hdev->dev, in hl_device_reset()
1017 hdev->disabled = true; in hl_device_reset()
1021 dev_err(hdev->dev, "Going to RESET device!\n"); in hl_device_reset()
1026 hdev->hard_reset_pending = true; in hl_device_reset()
1028 hdev->process_kill_trial_cnt = 0; in hl_device_reset()
1030 hdev->device_reset_work.fw_reset = fw_reset; in hl_device_reset()
1033 * Because the reset function can't run from heartbeat work, in hl_device_reset()
1034 * we need to call the reset function from a dedicated work. in hl_device_reset()
1036 queue_delayed_work(hdev->device_reset_work.wq, in hl_device_reset()
1037 &hdev->device_reset_work.reset_work, 0); in hl_device_reset()
1052 if (rc == -EBUSY) { in hl_device_reset()
1053 if (hdev->device_fini_pending) { in hl_device_reset()
1054 dev_crit(hdev->dev, in hl_device_reset()
1055 "Failed to kill all open processes, stopping hard reset\n"); in hl_device_reset()
1059 /* signal reset thread to reschedule */ in hl_device_reset()
1064 dev_crit(hdev->dev, in hl_device_reset()
1065 "Failed to kill all open processes, stopping hard reset\n"); in hl_device_reset()
1069 /* Flush the Event queue workers to make sure no other thread is in hl_device_reset()
1070 * reading or writing to registers during the reset in hl_device_reset()
1072 flush_workqueue(hdev->eq_wq); in hl_device_reset()
1075 /* Reset the H/W. It will be in idle state after this returns */ in hl_device_reset()
1076 hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset); in hl_device_reset()
1079 hdev->fw_loader.linux_loaded = false; in hl_device_reset()
1082 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1) in hl_device_reset()
1083 hdev->kernel_ctx = NULL; in hl_device_reset()
1087 hl_eq_reset(hdev, &hdev->event_queue); in hl_device_reset()
1090 /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */ in hl_device_reset()
1092 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in hl_device_reset()
1093 hl_cq_reset(hdev, &hdev->completion_queue[i]); in hl_device_reset()
1095 mutex_lock(&hdev->fpriv_list_lock); in hl_device_reset()
1098 if (hdev->compute_ctx) { in hl_device_reset()
1099 atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1); in hl_device_reset()
1100 hdev->compute_ctx->thread_ctx_switch_wait_token = 0; in hl_device_reset()
1103 mutex_unlock(&hdev->fpriv_list_lock); in hl_device_reset()
1105 /* Finished tear-down, starting to re-initialize */ in hl_device_reset()
1108 hdev->device_cpu_disabled = false; in hl_device_reset()
1109 hdev->hard_reset_pending = false; in hl_device_reset()
1111 if (hdev->kernel_ctx) { in hl_device_reset()
1112 dev_crit(hdev->dev, in hl_device_reset()
1113 "kernel ctx was alive during hard reset, something is terribly wrong\n"); in hl_device_reset()
1114 rc = -EBUSY; in hl_device_reset()
1120 dev_err(hdev->dev, in hl_device_reset()
1121 "Failed to initialize MMU S/W after hard reset\n"); in hl_device_reset()
1126 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), in hl_device_reset()
1128 if (!hdev->kernel_ctx) { in hl_device_reset()
1129 rc = -ENOMEM; in hl_device_reset()
1134 hdev->compute_ctx = NULL; in hl_device_reset()
1136 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); in hl_device_reset()
1138 dev_err(hdev->dev, in hl_device_reset()
1139 "failed to init kernel ctx in hard reset\n"); in hl_device_reset()
1140 kfree(hdev->kernel_ctx); in hl_device_reset()
1141 hdev->kernel_ctx = NULL; in hl_device_reset()
1151 hdev->disabled = false; in hl_device_reset()
1153 rc = hdev->asic_funcs->hw_init(hdev); in hl_device_reset()
1155 dev_err(hdev->dev, in hl_device_reset()
1156 "failed to initialize the H/W after reset\n"); in hl_device_reset()
1160 /* If device is not idle fail the reset process */ in hl_device_reset()
1161 if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask, in hl_device_reset()
1163 dev_err(hdev->dev, in hl_device_reset()
1164 "device is not idle (mask 0x%llx_%llx) after reset\n", in hl_device_reset()
1166 rc = -EIO; in hl_device_reset()
1171 rc = hdev->asic_funcs->test_queues(hdev); in hl_device_reset()
1173 dev_err(hdev->dev, in hl_device_reset()
1174 "Failed to detect if device is alive after reset\n"); in hl_device_reset()
1181 dev_err(hdev->dev, in hl_device_reset()
1182 "Failed late init after hard reset\n"); in hl_device_reset()
1188 dev_err(hdev->dev, in hl_device_reset()
1189 "Failed to init memory module after hard reset\n"); in hl_device_reset()
1195 rc = hdev->asic_funcs->soft_reset_late_init(hdev); in hl_device_reset()
1197 dev_err(hdev->dev, in hl_device_reset()
1198 "Failed late init after soft reset\n"); in hl_device_reset()
1203 atomic_set(&hdev->in_reset, 0); in hl_device_reset()
1204 hdev->needs_reset = false; in hl_device_reset()
1206 dev_notice(hdev->dev, "Successfully finished resetting the device\n"); in hl_device_reset()
1209 hdev->hard_reset_cnt++; in hl_device_reset()
1211 /* After reset is done, we are ready to receive events from in hl_device_reset()
1216 hdev->asic_funcs->enable_events_from_fw(hdev); in hl_device_reset()
1218 hdev->soft_reset_cnt++; in hl_device_reset()
1224 hdev->disabled = true; in hl_device_reset()
1227 dev_err(hdev->dev, in hl_device_reset()
1228 "Failed to reset! Device is NOT usable\n"); in hl_device_reset()
1229 hdev->hard_reset_cnt++; in hl_device_reset()
1231 dev_err(hdev->dev, in hl_device_reset()
1232 "Failed to do soft-reset, trying hard reset\n"); in hl_device_reset()
1233 hdev->soft_reset_cnt++; in hl_device_reset()
1238 atomic_set(&hdev->in_reset, 0); in hl_device_reset()
1244 * hl_device_init - main initialization function for habanalabs device
1258 name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2); in hl_device_init()
1260 rc = -ENOMEM; in hl_device_init()
1265 rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name, in hl_device_init()
1266 &hdev->cdev, &hdev->dev); in hl_device_init()
1273 name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2); in hl_device_init()
1275 rc = -ENOMEM; in hl_device_init()
1280 rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops, in hl_device_init()
1281 name, &hdev->cdev_ctrl, &hdev->dev_ctrl); in hl_device_init()
1288 /* Initialize ASIC function pointers and perform early init */ in hl_device_init()
1293 user_interrupt_cnt = hdev->asic_prop.user_interrupt_count; in hl_device_init()
1296 hdev->user_interrupt = kcalloc(user_interrupt_cnt, in hl_device_init()
1297 sizeof(*hdev->user_interrupt), in hl_device_init()
1300 if (!hdev->user_interrupt) { in hl_device_init()
1301 rc = -ENOMEM; in hl_device_init()
1308 * late init in hl_device_init()
1310 rc = hdev->asic_funcs->sw_init(hdev); in hl_device_init()
1325 dev_err(hdev->dev, "failed to initialize kernel queues\n"); in hl_device_init()
1329 cq_cnt = hdev->asic_prop.completion_queues_count; in hl_device_init()
1337 hdev->completion_queue = kcalloc(cq_cnt, in hl_device_init()
1338 sizeof(*hdev->completion_queue), in hl_device_init()
1341 if (!hdev->completion_queue) { in hl_device_init()
1342 dev_err(hdev->dev, in hl_device_init()
1344 rc = -ENOMEM; in hl_device_init()
1350 rc = hl_cq_init(hdev, &hdev->completion_queue[i], in hl_device_init()
1351 hdev->asic_funcs->get_queue_id_for_cq(hdev, i)); in hl_device_init()
1353 dev_err(hdev->dev, in hl_device_init()
1357 hdev->completion_queue[i].cq_idx = i; in hl_device_init()
1365 rc = hl_eq_init(hdev, &hdev->event_queue); in hl_device_init()
1367 dev_err(hdev->dev, "failed to initialize event queue\n"); in hl_device_init()
1374 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n"); in hl_device_init()
1379 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); in hl_device_init()
1380 if (!hdev->kernel_ctx) { in hl_device_init()
1381 rc = -ENOMEM; in hl_device_init()
1385 hdev->compute_ctx = NULL; in hl_device_init()
1387 hdev->asic_funcs->state_dump_init(hdev); in hl_device_init()
1394 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); in hl_device_init()
1396 dev_err(hdev->dev, "failed to initialize kernel context\n"); in hl_device_init()
1397 kfree(hdev->kernel_ctx); in hl_device_init()
1403 dev_err(hdev->dev, "failed to initialize CB pool\n"); in hl_device_init()
1418 hdev->disabled = false; in hl_device_init()
1420 rc = hdev->asic_funcs->hw_init(hdev); in hl_device_init()
1422 dev_err(hdev->dev, "failed to initialize the H/W\n"); in hl_device_init()
1428 rc = hdev->asic_funcs->test_queues(hdev); in hl_device_init()
1430 dev_err(hdev->dev, "Failed to detect if device is alive\n"); in hl_device_init()
1437 dev_err(hdev->dev, "Failed late initialization\n"); in hl_device_init()
1442 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", in hl_device_init()
1443 hdev->asic_name, in hl_device_init()
1444 hdev->asic_prop.dram_size / SZ_1G); in hl_device_init()
1448 dev_err(hdev->dev, "Failed to initialize memory module\n"); in hl_device_init()
1455 * From here there is no need to add char devices and create sysfs nodes in hl_device_init()
1461 dev_err(hdev->dev, in hl_device_init()
1468 * depending on card type for certain ASICs in hl_device_init()
1475 * hwmon-related sensors the device supports. in hl_device_init()
1480 dev_err(hdev->dev, "Failed to initialize hwmon\n"); in hl_device_init()
1485 dev_notice(hdev->dev, in hl_device_init()
1488 hdev->init_done = true; in hl_device_init()
1495 hdev->asic_funcs->enable_events_from_fw(hdev); in hl_device_init()
1500 if (hl_ctx_put(hdev->kernel_ctx) != 1) in hl_device_init()
1501 dev_err(hdev->dev, in hl_device_init()
1502 "kernel ctx is still alive on initialization failure\n"); in hl_device_init()
1508 hl_eq_fini(hdev, &hdev->event_queue); in hl_device_init()
1511 hl_cq_fini(hdev, &hdev->completion_queue[i]); in hl_device_init()
1512 kfree(hdev->completion_queue); in hl_device_init()
1516 hdev->asic_funcs->sw_fini(hdev); in hl_device_init()
1518 kfree(hdev->user_interrupt); in hl_device_init()
1522 put_device(hdev->dev_ctrl); in hl_device_init()
1524 put_device(hdev->dev); in hl_device_init()
1526 hdev->disabled = true; in hl_device_init()
1529 if (hdev->pdev) in hl_device_init()
1530 dev_err(&hdev->pdev->dev, in hl_device_init()
1532 hdev->id / 2); in hl_device_init()
1535 hdev->id / 2); in hl_device_init()
1541 * hl_device_fini - main tear-down function for habanalabs device
1553 dev_info(hdev->dev, "Removing device\n"); in hl_device_fini()
1555 hdev->device_fini_pending = 1; in hl_device_fini()
1556 flush_delayed_work(&hdev->device_reset_work.reset_work); in hl_device_fini()
1558 if (hdev->pldm) in hl_device_fini()
1564 * This function is competing with the reset function, so try to in hl_device_fini()
1565 * take the reset atomic and if we are already in middle of reset, in hl_device_fini()
1566 * wait until reset function is finished. Reset function is designed in hl_device_fini()
1568 * ports, the hard reset could take between 10-30 seconds in hl_device_fini()
1572 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); in hl_device_fini()
1575 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); in hl_device_fini()
1577 dev_crit(hdev->dev, in hl_device_fini()
1578 "Failed to remove device because reset function did not finish\n"); in hl_device_fini()
1584 * interrupts. We disable MSI/MSI-X at the halt_engines function and we in hl_device_fini()
1593 hdev->disabled = true; in hl_device_fini()
1597 hdev->hard_reset_pending = true; in hl_device_fini()
1607 dev_info(hdev->dev, in hl_device_fini()
1613 dev_crit(hdev->dev, "Failed to kill all open processes\n"); in hl_device_fini()
1619 /* Reset the H/W. It will be in idle state after this returns */ in hl_device_fini()
1620 hdev->asic_funcs->hw_fini(hdev, true, false); in hl_device_fini()
1622 hdev->fw_loader.linux_loaded = false; in hl_device_fini()
1625 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) in hl_device_fini()
1626 dev_err(hdev->dev, "kernel ctx is still alive\n"); in hl_device_fini()
1634 hl_eq_fini(hdev, &hdev->event_queue); in hl_device_fini()
1636 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in hl_device_fini()
1637 hl_cq_fini(hdev, &hdev->completion_queue[i]); in hl_device_fini()
1638 kfree(hdev->completion_queue); in hl_device_fini()
1639 kfree(hdev->user_interrupt); in hl_device_fini()
1644 hdev->asic_funcs->sw_fini(hdev); in hl_device_fini()
1659 * hl_rreg - Read an MMIO register
1669 return readl(hdev->rmmio + reg); in hl_rreg()
1673 * hl_wreg - Write to an MMIO register
1677 * @val: 32-bit value
1679 * Writes the 32-bit value into the MMIO register
1684 writel(val, hdev->rmmio + reg); in hl_wreg()