Lines Matching refs:hdev
46 static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region, in hl_set_dram_bar() argument
49 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_set_dram_bar()
58 old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr); in hl_set_dram_bar()
67 int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, in hl_access_sram_dram_region() argument
70 struct pci_mem_region *region = &hdev->pci_mem_region[region_type]; in hl_access_sram_dram_region()
75 old_base = hl_set_dram_bar(hdev, addr, region, &bar_region_base); in hl_access_sram_dram_region()
80 acc_addr = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + in hl_access_sram_dram_region()
105 rc = hl_set_dram_bar(hdev, old_base, region, NULL); in hl_access_sram_dram_region()
113 static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, in hl_dma_alloc_common() argument
121 ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag); in hl_dma_alloc_common()
124 ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle); in hl_dma_alloc_common()
129 trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size, in hl_dma_alloc_common()
135 static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr, in hl_asic_dma_free_common() argument
144 hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle); in hl_asic_dma_free_common()
147 hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle); in hl_asic_dma_free_common()
151 trace_habanalabs_dma_free(hdev->dev, store_cpu_addr, dma_handle, size, caller); in hl_asic_dma_free_common()
154 void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, in hl_asic_dma_alloc_coherent_caller() argument
157 return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller); in hl_asic_dma_alloc_coherent_caller()
160 void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, in hl_asic_dma_free_coherent_caller() argument
163 hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller); in hl_asic_dma_free_coherent_caller()
166 void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, in hl_asic_dma_pool_zalloc_caller() argument
169 return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller); in hl_asic_dma_pool_zalloc_caller()
172 void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, in hl_asic_dma_pool_free_caller() argument
175 hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller); in hl_asic_dma_pool_free_caller()
178 void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) in hl_cpu_accessible_dma_pool_alloc() argument
180 return hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); in hl_cpu_accessible_dma_pool_alloc()
183 void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) in hl_cpu_accessible_dma_pool_free() argument
185 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, vaddr); in hl_cpu_accessible_dma_pool_free()
188 int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) in hl_dma_map_sgtable() argument
190 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_dma_map_sgtable()
194 rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0); in hl_dma_map_sgtable()
206 void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) in hl_dma_unmap_sgtable() argument
208 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_dma_unmap_sgtable()
217 dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0); in hl_dma_unmap_sgtable()
228 int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, in hl_access_cfg_region() argument
231 struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG]; in hl_access_cfg_region()
235 dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32)); in hl_access_cfg_region()
257 dev_err(hdev->dev, "access type %d is not supported\n", acc_type); in hl_access_cfg_region()
273 int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, in hl_access_dev_mem() argument
278 return hl_access_cfg_region(hdev, addr, val, acc_type); in hl_access_dev_mem()
281 return hl_access_sram_dram_region(hdev, addr, val, acc_type, in hl_access_dev_mem()
314 enum hl_device_status hl_device_status(struct hl_device *hdev) in hl_device_status() argument
318 if (hdev->reset_info.in_reset) { in hl_device_status()
319 if (hdev->reset_info.in_compute_reset) in hl_device_status()
323 } else if (hdev->reset_info.needs_reset) { in hl_device_status()
325 } else if (hdev->disabled) { in hl_device_status()
327 } else if (!hdev->init_done) { in hl_device_status()
336 bool hl_device_operational(struct hl_device *hdev, in hl_device_operational() argument
341 current_status = hl_device_status(hdev); in hl_device_operational()
358 bool hl_ctrl_device_operational(struct hl_device *hdev, in hl_ctrl_device_operational() argument
363 current_status = hl_device_status(hdev); in hl_ctrl_device_operational()
380 static void print_idle_status_mask(struct hl_device *hdev, const char *message, in print_idle_status_mask() argument
384 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx_%016llx)\n", in print_idle_status_mask()
387 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx)\n", in print_idle_status_mask()
390 dev_err(hdev->dev, "%s (mask %#llx_%016llx)\n", in print_idle_status_mask()
393 dev_err(hdev->dev, "%s (mask %#llx)\n", message, idle_mask[0]); in print_idle_status_mask()
401 struct hl_device *hdev; in hpriv_release() local
405 hdev = hpriv->hdev; in hpriv_release()
407 hdev->asic_funcs->send_device_activity(hdev, false); in hpriv_release()
422 reset_device = hdev->reset_upon_device_release || hdev->reset_info.watchdog_active; in hpriv_release()
427 if (!hdev->reset_info.in_reset && !reset_device && hdev->pdev && !hdev->pldm) in hpriv_release()
428 device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask, in hpriv_release()
431 print_idle_status_mask(hdev, "device is not idle after user context is closed", in hpriv_release()
445 mutex_lock(&hdev->fpriv_list_lock); in hpriv_release()
447 mutex_unlock(&hdev->fpriv_list_lock); in hpriv_release()
450 hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE); in hpriv_release()
453 int rc = hdev->asic_funcs->scrub_device_mem(hdev); in hpriv_release()
456 dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc); in hpriv_release()
463 mutex_lock(&hdev->fpriv_list_lock); in hpriv_release()
464 hdev->is_compute_ctx_active = false; in hpriv_release()
465 mutex_unlock(&hdev->fpriv_list_lock); in hpriv_release()
467 hdev->compute_ctx_in_release = 0; in hpriv_release()
488 static void print_device_in_use_info(struct hl_device *hdev, const char *message) in print_device_in_use_info() argument
499 active_cs_num = hl_get_active_cs_num(hdev); in print_device_in_use_info()
505 dmabuf_export_cnt = atomic_read(&hdev->dmabuf_export_cnt); in print_device_in_use_info()
515 dev_notice(hdev->dev, "%s%s\n", message, buf); in print_device_in_use_info()
529 struct hl_device *hdev = hpriv->hdev; in hl_device_release() local
533 if (!hdev) { in hl_device_release()
539 hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr); in hl_device_release()
546 hdev->compute_ctx_in_release = 1; in hl_device_release()
549 print_device_in_use_info(hdev, "User process closed FD but device still in use"); in hl_device_release()
550 hl_device_reset(hdev, HL_DRV_RESET_HARD); in hl_device_release()
553 hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif; in hl_device_release()
561 struct hl_device *hdev = hpriv->hdev; in hl_device_release_ctrl() local
565 if (!hdev) { in hl_device_release_ctrl()
570 mutex_lock(&hdev->fpriv_ctrl_list_lock); in hl_device_release_ctrl()
572 mutex_unlock(&hdev->fpriv_ctrl_list_lock); in hl_device_release_ctrl()
598 struct hl_device *hdev = hpriv->hdev; in hl_mmap() local
601 if (!hdev) { in hl_mmap()
655 static int device_init_cdev(struct hl_device *hdev, struct class *class, in device_init_cdev() argument
668 (*dev)->devt = MKDEV(hdev->major, minor); in device_init_cdev()
671 dev_set_drvdata(*dev, hdev); in device_init_cdev()
677 static int cdev_sysfs_debugfs_add(struct hl_device *hdev) in cdev_sysfs_debugfs_add() argument
681 rc = cdev_device_add(&hdev->cdev, hdev->dev); in cdev_sysfs_debugfs_add()
683 dev_err(hdev->dev, in cdev_sysfs_debugfs_add()
688 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl); in cdev_sysfs_debugfs_add()
690 dev_err(hdev->dev, in cdev_sysfs_debugfs_add()
696 rc = hl_sysfs_init(hdev); in cdev_sysfs_debugfs_add()
698 dev_err(hdev->dev, "failed to initialize sysfs\n"); in cdev_sysfs_debugfs_add()
702 hl_debugfs_add_device(hdev); in cdev_sysfs_debugfs_add()
704 hdev->cdev_sysfs_debugfs_created = true; in cdev_sysfs_debugfs_add()
709 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); in cdev_sysfs_debugfs_add()
711 cdev_device_del(&hdev->cdev, hdev->dev); in cdev_sysfs_debugfs_add()
715 static void cdev_sysfs_debugfs_remove(struct hl_device *hdev) in cdev_sysfs_debugfs_remove() argument
717 if (!hdev->cdev_sysfs_debugfs_created) in cdev_sysfs_debugfs_remove()
720 hl_debugfs_remove_device(hdev); in cdev_sysfs_debugfs_remove()
721 hl_sysfs_fini(hdev); in cdev_sysfs_debugfs_remove()
722 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); in cdev_sysfs_debugfs_remove()
723 cdev_device_del(&hdev->cdev, hdev->dev); in cdev_sysfs_debugfs_remove()
726 put_device(hdev->dev); in cdev_sysfs_debugfs_remove()
727 put_device(hdev->dev_ctrl); in cdev_sysfs_debugfs_remove()
734 struct hl_device *hdev = device_reset_work->hdev; in device_hard_reset_pending() local
740 rc = hl_device_reset(hdev, flags); in device_hard_reset_pending()
742 if ((rc == -EBUSY) && !hdev->device_fini_pending) { in device_hard_reset_pending()
743 struct hl_ctx *ctx = hl_get_compute_ctx(hdev); in device_hard_reset_pending()
749 dev_info(hdev->dev, in device_hard_reset_pending()
754 dev_info(hdev->dev, "Could not reset device. will try again in %u seconds", in device_hard_reset_pending()
758 queue_delayed_work(hdev->reset_wq, &device_reset_work->reset_work, in device_hard_reset_pending()
767 struct hl_device *hdev = watchdog_work->hdev; in device_release_watchdog_func() local
770 dev_dbg(hdev->dev, "Device wasn't released in time. Initiate hard-reset.\n"); in device_release_watchdog_func()
774 hl_device_reset(hdev, flags); in device_release_watchdog_func()
785 static int device_early_init(struct hl_device *hdev) in device_early_init() argument
790 switch (hdev->asic_type) { in device_early_init()
792 goya_set_asic_funcs(hdev); in device_early_init()
793 strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name)); in device_early_init()
796 gaudi_set_asic_funcs(hdev); in device_early_init()
797 strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name)); in device_early_init()
800 gaudi_set_asic_funcs(hdev); in device_early_init()
801 strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name)); in device_early_init()
804 gaudi2_set_asic_funcs(hdev); in device_early_init()
805 strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name)); in device_early_init()
808 gaudi2_set_asic_funcs(hdev); in device_early_init()
809 strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name)); in device_early_init()
813 dev_err(hdev->dev, "Unrecognized ASIC type %d\n", in device_early_init()
814 hdev->asic_type); in device_early_init()
818 rc = hdev->asic_funcs->early_init(hdev); in device_early_init()
822 rc = hl_asid_init(hdev); in device_early_init()
826 if (hdev->asic_prop.completion_queues_count) { in device_early_init()
827 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count, in device_early_init()
830 if (!hdev->cq_wq) { in device_early_init()
836 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in device_early_init()
837 snprintf(workq_name, 32, "hl%u-free-jobs-%u", hdev->cdev_idx, (u32) i); in device_early_init()
838 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name); in device_early_init()
839 if (hdev->cq_wq[i] == NULL) { in device_early_init()
840 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n"); in device_early_init()
846 snprintf(workq_name, 32, "hl%u-events", hdev->cdev_idx); in device_early_init()
847 hdev->eq_wq = create_singlethread_workqueue(workq_name); in device_early_init()
848 if (hdev->eq_wq == NULL) { in device_early_init()
849 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n"); in device_early_init()
854 snprintf(workq_name, 32, "hl%u-cs-completions", hdev->cdev_idx); in device_early_init()
855 hdev->cs_cmplt_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0); in device_early_init()
856 if (!hdev->cs_cmplt_wq) { in device_early_init()
857 dev_err(hdev->dev, in device_early_init()
863 snprintf(workq_name, 32, "hl%u-ts-free-obj", hdev->cdev_idx); in device_early_init()
864 hdev->ts_free_obj_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0); in device_early_init()
865 if (!hdev->ts_free_obj_wq) { in device_early_init()
866 dev_err(hdev->dev, in device_early_init()
872 snprintf(workq_name, 32, "hl%u-prefetch", hdev->cdev_idx); in device_early_init()
873 hdev->prefetch_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0); in device_early_init()
874 if (!hdev->prefetch_wq) { in device_early_init()
875 dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n"); in device_early_init()
880 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), GFP_KERNEL); in device_early_init()
881 if (!hdev->hl_chip_info) { in device_early_init()
886 rc = hl_mmu_if_set_funcs(hdev); in device_early_init()
890 hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr); in device_early_init()
892 snprintf(workq_name, 32, "hl%u_device_reset", hdev->cdev_idx); in device_early_init()
893 hdev->reset_wq = create_singlethread_workqueue(workq_name); in device_early_init()
894 if (!hdev->reset_wq) { in device_early_init()
896 dev_err(hdev->dev, "Failed to create device reset WQ\n"); in device_early_init()
900 INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, device_hard_reset_pending); in device_early_init()
901 hdev->device_reset_work.hdev = hdev; in device_early_init()
902 hdev->device_fini_pending = 0; in device_early_init()
904 INIT_DELAYED_WORK(&hdev->device_release_watchdog_work.reset_work, in device_early_init()
906 hdev->device_release_watchdog_work.hdev = hdev; in device_early_init()
908 mutex_init(&hdev->send_cpu_message_lock); in device_early_init()
909 mutex_init(&hdev->debug_lock); in device_early_init()
910 INIT_LIST_HEAD(&hdev->cs_mirror_list); in device_early_init()
911 spin_lock_init(&hdev->cs_mirror_lock); in device_early_init()
912 spin_lock_init(&hdev->reset_info.lock); in device_early_init()
913 INIT_LIST_HEAD(&hdev->fpriv_list); in device_early_init()
914 INIT_LIST_HEAD(&hdev->fpriv_ctrl_list); in device_early_init()
915 mutex_init(&hdev->fpriv_list_lock); in device_early_init()
916 mutex_init(&hdev->fpriv_ctrl_list_lock); in device_early_init()
917 mutex_init(&hdev->clk_throttling.lock); in device_early_init()
922 hl_mem_mgr_fini(&hdev->kernel_mem_mgr); in device_early_init()
923 hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr); in device_early_init()
925 kfree(hdev->hl_chip_info); in device_early_init()
927 destroy_workqueue(hdev->prefetch_wq); in device_early_init()
929 destroy_workqueue(hdev->ts_free_obj_wq); in device_early_init()
931 destroy_workqueue(hdev->cs_cmplt_wq); in device_early_init()
933 destroy_workqueue(hdev->eq_wq); in device_early_init()
935 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in device_early_init()
936 if (hdev->cq_wq[i]) in device_early_init()
937 destroy_workqueue(hdev->cq_wq[i]); in device_early_init()
938 kfree(hdev->cq_wq); in device_early_init()
940 hl_asid_fini(hdev); in device_early_init()
942 if (hdev->asic_funcs->early_fini) in device_early_init()
943 hdev->asic_funcs->early_fini(hdev); in device_early_init()
954 static void device_early_fini(struct hl_device *hdev) in device_early_fini() argument
958 mutex_destroy(&hdev->debug_lock); in device_early_fini()
959 mutex_destroy(&hdev->send_cpu_message_lock); in device_early_fini()
961 mutex_destroy(&hdev->fpriv_list_lock); in device_early_fini()
962 mutex_destroy(&hdev->fpriv_ctrl_list_lock); in device_early_fini()
964 mutex_destroy(&hdev->clk_throttling.lock); in device_early_fini()
966 hl_mem_mgr_fini(&hdev->kernel_mem_mgr); in device_early_fini()
967 hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr); in device_early_fini()
969 kfree(hdev->hl_chip_info); in device_early_fini()
971 destroy_workqueue(hdev->prefetch_wq); in device_early_fini()
972 destroy_workqueue(hdev->ts_free_obj_wq); in device_early_fini()
973 destroy_workqueue(hdev->cs_cmplt_wq); in device_early_fini()
974 destroy_workqueue(hdev->eq_wq); in device_early_fini()
975 destroy_workqueue(hdev->reset_wq); in device_early_fini()
977 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in device_early_fini()
978 destroy_workqueue(hdev->cq_wq[i]); in device_early_fini()
979 kfree(hdev->cq_wq); in device_early_fini()
981 hl_asid_fini(hdev); in device_early_fini()
983 if (hdev->asic_funcs->early_fini) in device_early_fini()
984 hdev->asic_funcs->early_fini(hdev); in device_early_fini()
987 static bool is_pci_link_healthy(struct hl_device *hdev) in is_pci_link_healthy() argument
991 if (!hdev->pdev) in is_pci_link_healthy()
994 pci_read_config_word(hdev->pdev, PCI_VENDOR_ID, &vendor_id); in is_pci_link_healthy()
1001 struct hl_device *hdev = container_of(work, struct hl_device, in hl_device_heartbeat() local
1006 if (!hl_device_operational(hdev, NULL)) in hl_device_heartbeat()
1009 if (!hdev->asic_funcs->send_heartbeat(hdev)) in hl_device_heartbeat()
1012 if (hl_device_operational(hdev, NULL)) in hl_device_heartbeat()
1013 dev_err(hdev->dev, "Device heartbeat failed! PCI link is %s\n", in hl_device_heartbeat()
1014 is_pci_link_healthy(hdev) ? "healthy" : "broken"); in hl_device_heartbeat()
1018 hl_handle_fw_err(hdev, &info); in hl_device_heartbeat()
1019 hl_device_cond_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT, event_mask); in hl_device_heartbeat()
1034 if (!hdev->reset_info.in_reset) in hl_device_heartbeat()
1035 hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT; in hl_device_heartbeat()
1037 schedule_delayed_work(&hdev->work_heartbeat, in hl_device_heartbeat()
1049 static int device_late_init(struct hl_device *hdev) in device_late_init() argument
1053 if (hdev->asic_funcs->late_init) { in device_late_init()
1054 rc = hdev->asic_funcs->late_init(hdev); in device_late_init()
1056 dev_err(hdev->dev, in device_late_init()
1062 hdev->high_pll = hdev->asic_prop.high_pll; in device_late_init()
1064 if (hdev->heartbeat) { in device_late_init()
1065 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat); in device_late_init()
1066 schedule_delayed_work(&hdev->work_heartbeat, in device_late_init()
1070 hdev->late_init_done = true; in device_late_init()
1081 static void device_late_fini(struct hl_device *hdev) in device_late_fini() argument
1083 if (!hdev->late_init_done) in device_late_fini()
1086 if (hdev->heartbeat) in device_late_fini()
1087 cancel_delayed_work_sync(&hdev->work_heartbeat); in device_late_fini()
1089 if (hdev->asic_funcs->late_fini) in device_late_fini()
1090 hdev->asic_funcs->late_fini(hdev); in device_late_fini()
1092 hdev->late_init_done = false; in device_late_fini()
1095 int hl_device_utilization(struct hl_device *hdev, u32 *utilization) in hl_device_utilization() argument
1100 max_power = hdev->max_power; in hl_device_utilization()
1101 dc_power = hdev->asic_prop.dc_power_default; in hl_device_utilization()
1104 dev_warn(hdev->dev, "device utilization is not supported\n"); in hl_device_utilization()
1107 rc = hl_fw_cpucp_power_get(hdev, &curr_power); in hl_device_utilization()
1120 int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable) in hl_device_set_debug_mode() argument
1124 mutex_lock(&hdev->debug_lock); in hl_device_set_debug_mode()
1127 if (!hdev->in_debug) { in hl_device_set_debug_mode()
1128 dev_err(hdev->dev, in hl_device_set_debug_mode()
1134 if (!hdev->reset_info.hard_reset_pending) in hl_device_set_debug_mode()
1135 hdev->asic_funcs->halt_coresight(hdev, ctx); in hl_device_set_debug_mode()
1137 hdev->in_debug = 0; in hl_device_set_debug_mode()
1142 if (hdev->in_debug) { in hl_device_set_debug_mode()
1143 dev_err(hdev->dev, in hl_device_set_debug_mode()
1149 hdev->in_debug = 1; in hl_device_set_debug_mode()
1152 mutex_unlock(&hdev->debug_lock); in hl_device_set_debug_mode()
1157 static void take_release_locks(struct hl_device *hdev) in take_release_locks() argument
1162 hdev->asic_funcs->hw_queues_lock(hdev); in take_release_locks()
1163 hdev->asic_funcs->hw_queues_unlock(hdev); in take_release_locks()
1166 mutex_lock(&hdev->send_cpu_message_lock); in take_release_locks()
1167 mutex_unlock(&hdev->send_cpu_message_lock); in take_release_locks()
1170 mutex_lock(&hdev->fpriv_list_lock); in take_release_locks()
1171 mutex_unlock(&hdev->fpriv_list_lock); in take_release_locks()
1172 mutex_lock(&hdev->fpriv_ctrl_list_lock); in take_release_locks()
1173 mutex_unlock(&hdev->fpriv_ctrl_list_lock); in take_release_locks()
1176 static void hl_abort_waiting_for_completions(struct hl_device *hdev) in hl_abort_waiting_for_completions() argument
1178 hl_abort_waiting_for_cs_completions(hdev); in hl_abort_waiting_for_completions()
1183 hl_release_pending_user_interrupts(hdev); in hl_abort_waiting_for_completions()
1186 static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset, in cleanup_resources() argument
1190 device_late_fini(hdev); in cleanup_resources()
1197 hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset); in cleanup_resources()
1200 hl_cs_rollback_all(hdev, skip_wq_flush); in cleanup_resources()
1203 flush_workqueue(hdev->prefetch_wq); in cleanup_resources()
1205 hl_abort_waiting_for_completions(hdev); in cleanup_resources()
1217 int hl_device_suspend(struct hl_device *hdev) in hl_device_suspend() argument
1221 pci_save_state(hdev->pdev); in hl_device_suspend()
1224 spin_lock(&hdev->reset_info.lock); in hl_device_suspend()
1225 if (hdev->reset_info.in_reset) { in hl_device_suspend()
1226 spin_unlock(&hdev->reset_info.lock); in hl_device_suspend()
1227 dev_err(hdev->dev, "Can't suspend while in reset\n"); in hl_device_suspend()
1230 hdev->reset_info.in_reset = 1; in hl_device_suspend()
1231 spin_unlock(&hdev->reset_info.lock); in hl_device_suspend()
1234 hdev->disabled = true; in hl_device_suspend()
1236 take_release_locks(hdev); in hl_device_suspend()
1238 rc = hdev->asic_funcs->suspend(hdev); in hl_device_suspend()
1240 dev_err(hdev->dev, in hl_device_suspend()
1244 pci_disable_device(hdev->pdev); in hl_device_suspend()
1245 pci_set_power_state(hdev->pdev, PCI_D3hot); in hl_device_suspend()
1259 int hl_device_resume(struct hl_device *hdev) in hl_device_resume() argument
1263 pci_set_power_state(hdev->pdev, PCI_D0); in hl_device_resume()
1264 pci_restore_state(hdev->pdev); in hl_device_resume()
1265 rc = pci_enable_device_mem(hdev->pdev); in hl_device_resume()
1267 dev_err(hdev->dev, in hl_device_resume()
1272 pci_set_master(hdev->pdev); in hl_device_resume()
1274 rc = hdev->asic_funcs->resume(hdev); in hl_device_resume()
1276 dev_err(hdev->dev, "Failed to resume device after suspend\n"); in hl_device_resume()
1284 spin_lock(&hdev->reset_info.lock); in hl_device_resume()
1285 hdev->reset_info.in_reset = 0; in hl_device_resume()
1286 spin_unlock(&hdev->reset_info.lock); in hl_device_resume()
1288 rc = hl_device_reset(hdev, HL_DRV_RESET_HARD); in hl_device_resume()
1290 dev_err(hdev->dev, "Failed to reset device during resume\n"); in hl_device_resume()
1297 pci_disable_device(hdev->pdev); in hl_device_resume()
1302 static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev) in device_kill_open_processes() argument
1310 fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; in device_kill_open_processes()
1311 fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; in device_kill_open_processes()
1322 if (hdev->process_kill_trial_cnt) { in device_kill_open_processes()
1340 dev_info(hdev->dev, "Killing user process pid=%d\n", in device_kill_open_processes()
1352 dev_dbg(hdev->dev, in device_kill_open_processes()
1370 dev_dbg(hdev->dev, in device_kill_open_processes()
1383 if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS) in device_kill_open_processes()
1386 hdev->process_kill_trial_cnt++; in device_kill_open_processes()
1391 static void device_disable_open_processes(struct hl_device *hdev, bool control_dev) in device_disable_open_processes() argument
1397 fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; in device_disable_open_processes()
1398 fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; in device_disable_open_processes()
1402 hpriv->hdev = NULL; in device_disable_open_processes()
1406 static void send_disable_pci_access(struct hl_device *hdev, u32 flags) in send_disable_pci_access() argument
1422 if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) { in send_disable_pci_access()
1423 dev_warn(hdev->dev, "Failed to disable FW's PCI access\n"); in send_disable_pci_access()
1428 if (hdev->cpu_queues_enable) in send_disable_pci_access()
1429 synchronize_irq(pci_irq_vector(hdev->pdev, in send_disable_pci_access()
1430 hdev->asic_prop.eq_interrupt_id)); in send_disable_pci_access()
1434 static void handle_reset_trigger(struct hl_device *hdev, u32 flags) in handle_reset_trigger() argument
1439 if (hdev->is_compute_ctx_active) in handle_reset_trigger()
1449 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT; in handle_reset_trigger()
1452 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR; in handle_reset_trigger()
1455 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; in handle_reset_trigger()
1458 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; in handle_reset_trigger()
1466 if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) { in handle_reset_trigger()
1467 hdev->reset_info.prev_reset_trigger = cur_reset_trigger; in handle_reset_trigger()
1468 hdev->reset_info.reset_trigger_repeated = 0; in handle_reset_trigger()
1470 hdev->reset_info.reset_trigger_repeated = 1; in handle_reset_trigger()
1490 int hl_device_reset(struct hl_device *hdev, u32 flags) in hl_device_reset() argument
1498 if (!hdev->init_done) { in hl_device_reset()
1499 dev_err(hdev->dev, "Can't reset before initialization is done\n"); in hl_device_reset()
1509 reset_upon_device_release = hdev->reset_upon_device_release && from_dev_release; in hl_device_reset()
1511 if (!hard_reset && (hl_device_status(hdev) == HL_DEVICE_STATUS_MALFUNCTION)) { in hl_device_reset()
1512 dev_dbg(hdev->dev, "soft-reset isn't supported on a malfunctioning device\n"); in hl_device_reset()
1516 if (!hard_reset && !hdev->asic_prop.supports_compute_reset) { in hl_device_reset()
1517 dev_dbg(hdev->dev, "asic doesn't support compute reset - do hard-reset instead\n"); in hl_device_reset()
1523 dev_crit(hdev->dev, in hl_device_reset()
1531 if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) { in hl_device_reset()
1532 dev_dbg(hdev->dev, in hl_device_reset()
1539 if (from_hard_reset_thread && hdev->process_kill_trial_cnt) in hl_device_reset()
1549 spin_lock(&hdev->reset_info.lock); in hl_device_reset()
1550 if (hdev->reset_info.in_reset) { in hl_device_reset()
1552 if (hard_reset && hdev->reset_info.in_compute_reset) in hl_device_reset()
1553 hdev->reset_info.hard_reset_schedule_flags = flags; in hl_device_reset()
1554 spin_unlock(&hdev->reset_info.lock); in hl_device_reset()
1561 hdev->reset_info.in_compute_reset = !hard_reset; in hl_device_reset()
1563 hdev->reset_info.in_reset = 1; in hl_device_reset()
1565 spin_unlock(&hdev->reset_info.lock); in hl_device_reset()
1571 if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) { in hl_device_reset()
1573 &hdev->device_release_watchdog_work; in hl_device_reset()
1575 hdev->reset_info.watchdog_active = 0; in hl_device_reset()
1580 hdev->reset_info.in_compute_reset = 0; in hl_device_reset()
1591 handle_reset_trigger(hdev, flags); in hl_device_reset()
1592 send_disable_pci_access(hdev, flags); in hl_device_reset()
1595 hdev->disabled = true; in hl_device_reset()
1597 take_release_locks(hdev); in hl_device_reset()
1600 dev_info(hdev->dev, "Going to reset device\n"); in hl_device_reset()
1602 dev_dbg(hdev->dev, "Going to reset device after release by user\n"); in hl_device_reset()
1604 dev_dbg(hdev->dev, "Going to reset engines of inference device\n"); in hl_device_reset()
1608 hdev->reset_info.hard_reset_pending = true; in hl_device_reset()
1610 hdev->process_kill_trial_cnt = 0; in hl_device_reset()
1612 hdev->device_reset_work.flags = flags; in hl_device_reset()
1618 queue_delayed_work(hdev->reset_wq, &hdev->device_reset_work.reset_work, 0); in hl_device_reset()
1623 cleanup_resources(hdev, hard_reset, fw_reset, from_dev_release); in hl_device_reset()
1631 rc = device_kill_open_processes(hdev, 0, false); in hl_device_reset()
1634 if (hdev->device_fini_pending) { in hl_device_reset()
1635 dev_crit(hdev->dev, in hl_device_reset()
1637 dev_name(&(hdev)->pdev->dev)); in hl_device_reset()
1646 dev_crit(hdev->dev, in hl_device_reset()
1648 dev_name(&(hdev)->pdev->dev)); in hl_device_reset()
1655 flush_workqueue(hdev->eq_wq); in hl_device_reset()
1659 hw_fini_rc = hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset); in hl_device_reset()
1662 hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; in hl_device_reset()
1665 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1) in hl_device_reset()
1666 hdev->kernel_ctx = NULL; in hl_device_reset()
1668 hl_vm_fini(hdev); in hl_device_reset()
1669 hl_mmu_fini(hdev); in hl_device_reset()
1670 hl_eq_reset(hdev, &hdev->event_queue); in hl_device_reset()
1674 hl_hw_queue_reset(hdev, hard_reset); in hl_device_reset()
1675 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in hl_device_reset()
1676 hl_cq_reset(hdev, &hdev->completion_queue[i]); in hl_device_reset()
1679 ctx = hl_get_compute_ctx(hdev); in hl_device_reset()
1693 hdev->device_cpu_disabled = false; in hl_device_reset()
1694 hdev->reset_info.hard_reset_pending = false; in hl_device_reset()
1696 if (hdev->reset_info.reset_trigger_repeated && in hl_device_reset()
1697 (hdev->reset_info.prev_reset_trigger == in hl_device_reset()
1702 dev_crit(hdev->dev, in hl_device_reset()
1704 dev_name(&(hdev)->pdev->dev)); in hl_device_reset()
1709 if (hdev->kernel_ctx) { in hl_device_reset()
1710 dev_crit(hdev->dev, in hl_device_reset()
1712 dev_name(&(hdev)->pdev->dev)); in hl_device_reset()
1717 rc = hl_mmu_init(hdev); in hl_device_reset()
1719 dev_err(hdev->dev, in hl_device_reset()
1725 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), in hl_device_reset()
1727 if (!hdev->kernel_ctx) { in hl_device_reset()
1729 hl_mmu_fini(hdev); in hl_device_reset()
1733 hdev->is_compute_ctx_active = false; in hl_device_reset()
1735 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); in hl_device_reset()
1737 dev_err(hdev->dev, in hl_device_reset()
1739 kfree(hdev->kernel_ctx); in hl_device_reset()
1740 hdev->kernel_ctx = NULL; in hl_device_reset()
1741 hl_mmu_fini(hdev); in hl_device_reset()
1750 hdev->disabled = false; in hl_device_reset()
1754 rc = hl_fw_read_preboot_status(hdev); in hl_device_reset()
1759 rc = hdev->asic_funcs->hw_init(hdev); in hl_device_reset()
1761 dev_err(hdev->dev, "failed to initialize the H/W after reset\n"); in hl_device_reset()
1766 if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask, in hl_device_reset()
1768 print_idle_status_mask(hdev, "device is not idle after reset", idle_mask); in hl_device_reset()
1774 rc = hdev->asic_funcs->test_queues(hdev); in hl_device_reset()
1776 dev_err(hdev->dev, "Failed to detect if device is alive after reset\n"); in hl_device_reset()
1781 rc = device_late_init(hdev); in hl_device_reset()
1783 dev_err(hdev->dev, "Failed late init after hard reset\n"); in hl_device_reset()
1787 rc = hl_vm_init(hdev); in hl_device_reset()
1789 dev_err(hdev->dev, "Failed to init memory module after hard reset\n"); in hl_device_reset()
1793 if (!hdev->asic_prop.fw_security_enabled) in hl_device_reset()
1794 hl_fw_set_max_power(hdev); in hl_device_reset()
1796 rc = hdev->asic_funcs->compute_reset_late_init(hdev); in hl_device_reset()
1799 dev_err(hdev->dev, in hl_device_reset()
1802 dev_err(hdev->dev, "Failed late init after compute reset\n"); in hl_device_reset()
1807 rc = hdev->asic_funcs->scrub_device_mem(hdev); in hl_device_reset()
1809 dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc); in hl_device_reset()
1813 spin_lock(&hdev->reset_info.lock); in hl_device_reset()
1814 hdev->reset_info.in_compute_reset = 0; in hl_device_reset()
1820 if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags) in hl_device_reset()
1823 hdev->reset_info.in_reset = 0; in hl_device_reset()
1825 spin_unlock(&hdev->reset_info.lock); in hl_device_reset()
1827 hdev->reset_info.needs_reset = false; in hl_device_reset()
1830 dev_info(hdev->dev, in hl_device_reset()
1832 dev_name(&(hdev)->pdev->dev)); in hl_device_reset()
1834 dev_dbg(hdev->dev, in hl_device_reset()
1836 dev_name(&(hdev)->pdev->dev)); in hl_device_reset()
1839 hdev->reset_info.hard_reset_cnt++; in hl_device_reset()
1846 hdev->asic_funcs->enable_events_from_fw(hdev); in hl_device_reset()
1849 hdev->reset_info.compute_reset_cnt++; in hl_device_reset()
1852 dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n"); in hl_device_reset()
1853 flags = hdev->reset_info.hard_reset_schedule_flags; in hl_device_reset()
1854 hdev->reset_info.hard_reset_schedule_flags = 0; in hl_device_reset()
1863 hdev->disabled = true; in hl_device_reset()
1865 spin_lock(&hdev->reset_info.lock); in hl_device_reset()
1866 hdev->reset_info.in_compute_reset = 0; in hl_device_reset()
1869 dev_err(hdev->dev, in hl_device_reset()
1871 dev_name(&(hdev)->pdev->dev)); in hl_device_reset()
1872 hdev->reset_info.hard_reset_cnt++; in hl_device_reset()
1875 dev_err(hdev->dev, "Failed to reset device after user release\n"); in hl_device_reset()
1878 dev_err(hdev->dev, "Failed to do compute reset\n"); in hl_device_reset()
1879 hdev->reset_info.compute_reset_cnt++; in hl_device_reset()
1882 spin_unlock(&hdev->reset_info.lock); in hl_device_reset()
1888 hdev->reset_info.in_reset = 0; in hl_device_reset()
1890 spin_unlock(&hdev->reset_info.lock); in hl_device_reset()
1904 int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask) in hl_device_cond_reset() argument
1914 dev_err(hdev->dev, "Resetting device without a reset indication to user\n"); in hl_device_cond_reset()
1918 ctx = hl_get_compute_ctx(hdev); in hl_device_cond_reset()
1925 spin_lock(&hdev->reset_info.lock); in hl_device_cond_reset()
1926 if (hdev->reset_info.in_reset) { in hl_device_cond_reset()
1927 spin_unlock(&hdev->reset_info.lock); in hl_device_cond_reset()
1931 if (hdev->reset_info.watchdog_active) in hl_device_cond_reset()
1934 hdev->device_release_watchdog_work.flags = flags; in hl_device_cond_reset()
1935 dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n", in hl_device_cond_reset()
1936 hdev->device_release_watchdog_timeout_sec); in hl_device_cond_reset()
1937 schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work, in hl_device_cond_reset()
1938 msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000)); in hl_device_cond_reset()
1939 hdev->reset_info.watchdog_active = 1; in hl_device_cond_reset()
1941 spin_unlock(&hdev->reset_info.lock); in hl_device_cond_reset()
1943 hl_notifier_event_send_all(hdev, event_mask); in hl_device_cond_reset()
1947 hl_abort_waiting_for_completions(hdev); in hl_device_cond_reset()
1953 hl_notifier_event_send_all(hdev, event_mask); in hl_device_cond_reset()
1957 return hl_device_reset(hdev, flags); in hl_device_cond_reset()
1978 void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask) in hl_notifier_event_send_all() argument
1983 dev_warn(hdev->dev, "Skip sending zero event"); in hl_notifier_event_send_all()
1987 mutex_lock(&hdev->fpriv_list_lock); in hl_notifier_event_send_all()
1989 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) in hl_notifier_event_send_all()
1992 mutex_unlock(&hdev->fpriv_list_lock); in hl_notifier_event_send_all()
1995 mutex_lock(&hdev->fpriv_ctrl_list_lock); in hl_notifier_event_send_all()
1997 list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node) in hl_notifier_event_send_all()
2000 mutex_unlock(&hdev->fpriv_ctrl_list_lock); in hl_notifier_event_send_all()
2003 static int create_cdev(struct hl_device *hdev) in create_cdev() argument
2008 hdev->cdev_idx = hdev->id / 2; in create_cdev()
2010 name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx); in create_cdev()
2017 rc = device_init_cdev(hdev, hdev->hclass, hdev->id, &hl_ops, name, in create_cdev()
2018 &hdev->cdev, &hdev->dev); in create_cdev()
2025 name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx); in create_cdev()
2032 rc = device_init_cdev(hdev, hdev->hclass, hdev->id_control, &hl_ctrl_ops, in create_cdev()
2033 name, &hdev->cdev_ctrl, &hdev->dev_ctrl); in create_cdev()
2043 put_device(hdev->dev); in create_cdev()
2057 int hl_device_init(struct hl_device *hdev) in hl_device_init() argument
2062 rc = create_cdev(hdev); in hl_device_init()
2067 rc = device_early_init(hdev); in hl_device_init()
2071 user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count + in hl_device_init()
2072 hdev->asic_prop.user_interrupt_count; in hl_device_init()
2075 hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt), in hl_device_init()
2077 if (!hdev->user_interrupt) { in hl_device_init()
2087 rc = hdev->asic_funcs->sw_init(hdev); in hl_device_init()
2093 hl_multi_cs_completion_init(hdev); in hl_device_init()
2100 rc = hl_hw_queues_create(hdev); in hl_device_init()
2102 dev_err(hdev->dev, "failed to initialize kernel queues\n"); in hl_device_init()
2106 cq_cnt = hdev->asic_prop.completion_queues_count; in hl_device_init()
2114 hdev->completion_queue = kcalloc(cq_cnt, in hl_device_init()
2115 sizeof(*hdev->completion_queue), in hl_device_init()
2118 if (!hdev->completion_queue) { in hl_device_init()
2119 dev_err(hdev->dev, in hl_device_init()
2127 rc = hl_cq_init(hdev, &hdev->completion_queue[i], in hl_device_init()
2128 hdev->asic_funcs->get_queue_id_for_cq(hdev, i)); in hl_device_init()
2130 dev_err(hdev->dev, in hl_device_init()
2134 hdev->completion_queue[i].cq_idx = i; in hl_device_init()
2137 hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs, in hl_device_init()
2139 if (!hdev->shadow_cs_queue) { in hl_device_init()
2149 rc = hl_eq_init(hdev, &hdev->event_queue); in hl_device_init()
2151 dev_err(hdev->dev, "failed to initialize event queue\n"); in hl_device_init()
2156 rc = hl_mmu_init(hdev); in hl_device_init()
2158 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n"); in hl_device_init()
2163 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); in hl_device_init()
2164 if (!hdev->kernel_ctx) { in hl_device_init()
2169 hdev->is_compute_ctx_active = false; in hl_device_init()
2171 hdev->asic_funcs->state_dump_init(hdev); in hl_device_init()
2173 hdev->device_release_watchdog_timeout_sec = HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC; in hl_device_init()
2175 hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL; in hl_device_init()
2177 rc = hl_debugfs_device_init(hdev); in hl_device_init()
2179 dev_err(hdev->dev, "failed to initialize debugfs entry structure\n"); in hl_device_init()
2180 kfree(hdev->kernel_ctx); in hl_device_init()
2187 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); in hl_device_init()
2189 dev_err(hdev->dev, "failed to initialize kernel context\n"); in hl_device_init()
2190 kfree(hdev->kernel_ctx); in hl_device_init()
2194 rc = hl_cb_pool_init(hdev); in hl_device_init()
2196 dev_err(hdev->dev, "failed to initialize CB pool\n"); in hl_device_init()
2200 rc = hl_dec_init(hdev); in hl_device_init()
2202 dev_err(hdev->dev, "Failed to initialize the decoder module\n"); in hl_device_init()
2216 hdev->disabled = false; in hl_device_init()
2218 rc = hdev->asic_funcs->hw_init(hdev); in hl_device_init()
2220 dev_err(hdev->dev, "failed to initialize the H/W\n"); in hl_device_init()
2226 rc = hdev->asic_funcs->test_queues(hdev); in hl_device_init()
2228 dev_err(hdev->dev, "Failed to detect if device is alive\n"); in hl_device_init()
2233 rc = device_late_init(hdev); in hl_device_init()
2235 dev_err(hdev->dev, "Failed late initialization\n"); in hl_device_init()
2240 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", in hl_device_init()
2241 hdev->asic_name, in hl_device_init()
2242 hdev->asic_prop.dram_size / SZ_1G); in hl_device_init()
2244 rc = hl_vm_init(hdev); in hl_device_init()
2246 dev_err(hdev->dev, "Failed to initialize memory module\n"); in hl_device_init()
2256 rc = cdev_sysfs_debugfs_add(hdev); in hl_device_init()
2258 dev_err(hdev->dev, "Failed to add char devices and sysfs/debugfs files\n"); in hl_device_init()
2266 if (hdev->asic_prop.set_max_power_on_device_init && in hl_device_init()
2267 !hdev->asic_prop.fw_security_enabled) in hl_device_init()
2268 hl_fw_set_max_power(hdev); in hl_device_init()
2276 rc = hl_hwmon_init(hdev); in hl_device_init()
2278 dev_err(hdev->dev, "Failed to initialize hwmon\n"); in hl_device_init()
2283 dev_notice(hdev->dev, in hl_device_init()
2285 dev_name(&(hdev)->pdev->dev)); in hl_device_init()
2287 hdev->init_done = true; in hl_device_init()
2294 hdev->asic_funcs->enable_events_from_fw(hdev); in hl_device_init()
2299 hl_cb_pool_fini(hdev); in hl_device_init()
2301 if (hl_ctx_put(hdev->kernel_ctx) != 1) in hl_device_init()
2302 dev_err(hdev->dev, in hl_device_init()
2305 hl_debugfs_device_fini(hdev); in hl_device_init()
2307 hl_mmu_fini(hdev); in hl_device_init()
2309 hl_eq_fini(hdev, &hdev->event_queue); in hl_device_init()
2311 kfree(hdev->shadow_cs_queue); in hl_device_init()
2314 hl_cq_fini(hdev, &hdev->completion_queue[i]); in hl_device_init()
2315 kfree(hdev->completion_queue); in hl_device_init()
2317 hl_hw_queues_destroy(hdev); in hl_device_init()
2319 hdev->asic_funcs->sw_fini(hdev); in hl_device_init()
2321 kfree(hdev->user_interrupt); in hl_device_init()
2323 device_early_fini(hdev); in hl_device_init()
2325 put_device(hdev->dev_ctrl); in hl_device_init()
2326 put_device(hdev->dev); in hl_device_init()
2328 hdev->disabled = true; in hl_device_init()
2330 cdev_sysfs_debugfs_add(hdev); in hl_device_init()
2331 dev_err(&hdev->pdev->dev, in hl_device_init()
2333 hdev->cdev_idx, dev_name(&hdev->pdev->dev)); in hl_device_init()
2345 void hl_device_fini(struct hl_device *hdev) in hl_device_fini() argument
2352 dev_info(hdev->dev, "Removing device\n"); in hl_device_fini()
2354 hdev->device_fini_pending = 1; in hl_device_fini()
2355 flush_delayed_work(&hdev->device_reset_work.reset_work); in hl_device_fini()
2357 if (hdev->pldm) in hl_device_fini()
2372 spin_lock(&hdev->reset_info.lock); in hl_device_fini()
2373 device_in_reset = !!hdev->reset_info.in_reset; in hl_device_fini()
2375 hdev->reset_info.in_reset = 1; in hl_device_fini()
2376 spin_unlock(&hdev->reset_info.lock); in hl_device_fini()
2381 spin_lock(&hdev->reset_info.lock); in hl_device_fini()
2382 device_in_reset = !!hdev->reset_info.in_reset; in hl_device_fini()
2384 hdev->reset_info.in_reset = 1; in hl_device_fini()
2385 spin_unlock(&hdev->reset_info.lock); in hl_device_fini()
2388 dev_crit(hdev->dev, in hl_device_fini()
2390 dev_name(&(hdev)->pdev->dev)); in hl_device_fini()
2395 cancel_delayed_work_sync(&hdev->device_release_watchdog_work.reset_work); in hl_device_fini()
2404 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); in hl_device_fini()
2407 hdev->disabled = true; in hl_device_fini()
2409 take_release_locks(hdev); in hl_device_fini()
2411 hdev->reset_info.hard_reset_pending = true; in hl_device_fini()
2413 hl_hwmon_fini(hdev); in hl_device_fini()
2415 cleanup_resources(hdev, true, false, false); in hl_device_fini()
2421 dev_info(hdev->dev, in hl_device_fini()
2425 hdev->process_kill_trial_cnt = 0; in hl_device_fini()
2426 rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false); in hl_device_fini()
2428 dev_crit(hdev->dev, "Failed to kill all open processes\n"); in hl_device_fini()
2429 device_disable_open_processes(hdev, false); in hl_device_fini()
2432 hdev->process_kill_trial_cnt = 0; in hl_device_fini()
2433 rc = device_kill_open_processes(hdev, 0, true); in hl_device_fini()
2435 dev_crit(hdev->dev, "Failed to kill all control device open processes\n"); in hl_device_fini()
2436 device_disable_open_processes(hdev, true); in hl_device_fini()
2439 hl_cb_pool_fini(hdev); in hl_device_fini()
2442 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in hl_device_fini()
2444 dev_err(hdev->dev, "hw_fini failed in device fini while removing device %d\n", rc); in hl_device_fini()
2446 hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; in hl_device_fini()
2449 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) in hl_device_fini()
2450 dev_err(hdev->dev, "kernel ctx is still alive\n"); in hl_device_fini()
2452 hl_dec_fini(hdev); in hl_device_fini()
2454 hl_vm_fini(hdev); in hl_device_fini()
2456 hl_mmu_fini(hdev); in hl_device_fini()
2458 vfree(hdev->captured_err_info.page_fault_info.user_mappings); in hl_device_fini()
2460 hl_eq_fini(hdev, &hdev->event_queue); in hl_device_fini()
2462 kfree(hdev->shadow_cs_queue); in hl_device_fini()
2464 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in hl_device_fini()
2465 hl_cq_fini(hdev, &hdev->completion_queue[i]); in hl_device_fini()
2466 kfree(hdev->completion_queue); in hl_device_fini()
2467 kfree(hdev->user_interrupt); in hl_device_fini()
2469 hl_hw_queues_destroy(hdev); in hl_device_fini()
2472 hdev->asic_funcs->sw_fini(hdev); in hl_device_fini()
2474 device_early_fini(hdev); in hl_device_fini()
2477 cdev_sysfs_debugfs_remove(hdev); in hl_device_fini()
2479 hl_debugfs_device_fini(hdev); in hl_device_fini()
2497 inline u32 hl_rreg(struct hl_device *hdev, u32 reg) in hl_rreg() argument
2499 u32 val = readl(hdev->rmmio + reg); in hl_rreg()
2502 trace_habanalabs_rreg32(hdev->dev, reg, val); in hl_rreg()
2517 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val) in hl_wreg() argument
2520 trace_habanalabs_wreg32(hdev->dev, reg, val); in hl_wreg()
2522 writel(val, hdev->rmmio + reg); in hl_wreg()
2525 void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, in hl_capture_razwi() argument
2528 struct razwi_info *razwi_info = &hdev->captured_err_info.razwi_info; in hl_capture_razwi()
2531 dev_err(hdev->dev, in hl_capture_razwi()
2538 if (atomic_cmpxchg(&hdev->captured_err_info.razwi_info.razwi_detected, 0, 1)) in hl_capture_razwi()
2551 void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, in hl_handle_razwi() argument
2554 hl_capture_razwi(hdev, addr, engine_id, num_of_engines, flags); in hl_handle_razwi()
2560 static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu) in hl_capture_user_mappings() argument
2562 struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info; in hl_capture_user_mappings()
2574 ctx = hl_get_compute_ctx(hdev); in hl_capture_user_mappings()
2576 dev_err(hdev->dev, "Can't get user context for user mappings\n"); in hl_capture_user_mappings()
2622 void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu) in hl_capture_page_fault() argument
2624 struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info; in hl_capture_page_fault()
2633 hl_capture_user_mappings(hdev, is_pmmu); in hl_capture_page_fault()
2638 void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu, in hl_handle_page_fault() argument
2641 hl_capture_page_fault(hdev, addr, eng_id, is_pmmu); in hl_handle_page_fault()
2647 static void hl_capture_hw_err(struct hl_device *hdev, u16 event_id) in hl_capture_hw_err() argument
2649 struct hw_err_info *info = &hdev->captured_err_info.hw_err; in hl_capture_hw_err()
2661 void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask) in hl_handle_critical_hw_err() argument
2663 hl_capture_hw_err(hdev, event_id); in hl_handle_critical_hw_err()
2669 static void hl_capture_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *fw_info) in hl_capture_fw_err() argument
2671 struct fw_err_info *info = &hdev->captured_err_info.fw_err; in hl_capture_fw_err()
2685 void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info) in hl_handle_fw_err() argument
2687 hl_capture_fw_err(hdev, info); in hl_handle_fw_err()