500 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_set_hbm_bar_base()  local
505 	if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr))  in gaudi_set_hbm_bar_base()
516 	if (gaudi) {  in gaudi_set_hbm_bar_base()
517 		old_addr = gaudi->hbm_bar_cur_addr;  in gaudi_set_hbm_bar_base()
518 		gaudi->hbm_bar_cur_addr = addr;  in gaudi_set_hbm_bar_base()
791 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_late_init()  local
794 	rc = gaudi->cpucp_info_get(hdev);  in gaudi_late_init()
909 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_free_internal_qmans_pq_mem()  local
914 		q = &gaudi->internal_qmans[i];  in gaudi_free_internal_qmans_pq_mem()
925 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_alloc_internal_qmans_pq_mem()  local
933 		q = &gaudi->internal_qmans[i];  in gaudi_alloc_internal_qmans_pq_mem()
971 	struct gaudi_device *gaudi;  in gaudi_sw_init()  local
976 	gaudi = kzalloc(sizeof(*gaudi), GFP_KERNEL);  in gaudi_sw_init()
977 	if (!gaudi)  in gaudi_sw_init()
990 			gaudi->events[event_id++] =  in gaudi_sw_init()
995 	gaudi->cpucp_info_get = gaudi_cpucp_info_get;  in gaudi_sw_init()
997 	gaudi->max_freq_value = GAUDI_MAX_CLK_FREQ;  in gaudi_sw_init()
999 	hdev->asic_specific = gaudi;  in gaudi_sw_init()
1036 	spin_lock_init(&gaudi->hw_queues_lock);  in gaudi_sw_init()
1037 	mutex_init(&gaudi->clk_gate_mutex);  in gaudi_sw_init()
1056 	kfree(gaudi);  in gaudi_sw_init()
1062 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_sw_fini()  local
1077 	mutex_destroy(&gaudi->clk_gate_mutex);  in gaudi_sw_fini()
1079 	kfree(gaudi);  in gaudi_sw_fini()
1169 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_enable_msi()  local
1172 	if (gaudi->hw_cap_initialized & HW_CAP_MSI)  in gaudi_enable_msi()
1183 		gaudi->multi_msi_mode = false;  in gaudi_enable_msi()
1186 		gaudi->multi_msi_mode = true;  in gaudi_enable_msi()
1193 	gaudi->hw_cap_initialized |= HW_CAP_MSI;  in gaudi_enable_msi()
1204 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_sync_irqs()  local
1207 	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))  in gaudi_sync_irqs()
1211 	if (gaudi->multi_msi_mode) {  in gaudi_sync_irqs()
1225 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_msi()  local
1228 	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))  in gaudi_disable_msi()
1233 	if (gaudi->multi_msi_mode) {  in gaudi_disable_msi()
1248 	gaudi->hw_cap_initialized &= ~HW_CAP_MSI;  in gaudi_disable_msi()
1253 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_scrambler_sram()  local
1255 	if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER)  in gaudi_init_scrambler_sram()
1312 	gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER;  in gaudi_init_scrambler_sram()
1317 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_scrambler_hbm()  local
1319 	if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER)  in gaudi_init_scrambler_hbm()
1376 	gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER;  in gaudi_init_scrambler_hbm()
1959 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_pci_dma_qmans()  local
1963 	if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)  in gaudi_init_pci_dma_qmans()
1995 	gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA;  in gaudi_init_pci_dma_qmans()
2081 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_hbm_dma_qmans()  local
2086 	if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)  in gaudi_init_hbm_dma_qmans()
2099 			q = &gaudi->internal_qmans[internal_q_index];  in gaudi_init_hbm_dma_qmans()
2113 	gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA;  in gaudi_init_hbm_dma_qmans()
2199 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_mme_qmans()  local
2205 	if (gaudi->hw_cap_initialized & HW_CAP_MME)  in gaudi_init_mme_qmans()
2217 		q = &gaudi->internal_qmans[internal_q_index];  in gaudi_init_mme_qmans()
2233 	gaudi->hw_cap_initialized |= HW_CAP_MME;  in gaudi_init_mme_qmans()
2320 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_tpc_qmans()  local
2328 	if (gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)  in gaudi_init_tpc_qmans()
2338 			q = &gaudi->internal_qmans[internal_q_index];  in gaudi_init_tpc_qmans()
2358 		gaudi->hw_cap_initialized |=  in gaudi_init_tpc_qmans()
2365 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_pci_dma_qmans()  local
2367 	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))  in gaudi_disable_pci_dma_qmans()
2377 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_hbm_dma_qmans()  local
2379 	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))  in gaudi_disable_hbm_dma_qmans()
2391 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_mme_qmans()  local
2393 	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))  in gaudi_disable_mme_qmans()
2402 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_tpc_qmans()  local
2406 	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi_disable_tpc_qmans()
2417 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_pci_dma_qmans()  local
2419 	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))  in gaudi_stop_pci_dma_qmans()
2430 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_hbm_dma_qmans()  local
2432 	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))  in gaudi_stop_hbm_dma_qmans()
2446 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_mme_qmans()  local
2448 	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))  in gaudi_stop_mme_qmans()
2458 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_tpc_qmans()  local
2460 	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi_stop_tpc_qmans()
2475 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_pci_dma_stall()  local
2477 	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))  in gaudi_pci_dma_stall()
2487 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hbm_dma_stall()  local
2489 	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))  in gaudi_hbm_dma_stall()
2501 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mme_stall()  local
2503 	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))  in gaudi_mme_stall()
2527 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_tpc_stall()  local
2529 	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi_tpc_stall()
2544 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_set_clock_gating()  local
2597 	gaudi->hw_cap_initialized |= HW_CAP_CLK_GATE;  in gaudi_set_clock_gating()
2602 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_clock_gating()  local
2606 	if (!(gaudi->hw_cap_initialized & HW_CAP_CLK_GATE))  in gaudi_disable_clock_gating()
2628 	gaudi->hw_cap_initialized &= ~(HW_CAP_CLK_GATE);  in gaudi_disable_clock_gating()
2692 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_init()  local
2699 	if (gaudi->hw_cap_initialized & HW_CAP_MMU)  in gaudi_mmu_init()
2732 	gaudi->mmu_cache_inv_pi = 1;  in gaudi_mmu_init()
2734 	gaudi->hw_cap_initialized |= HW_CAP_MMU;  in gaudi_mmu_init()
2800 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_cpu()  local
2806 	if (gaudi->hw_cap_initialized & HW_CAP_CPU)  in gaudi_init_cpu()
2825 	gaudi->hw_cap_initialized |= HW_CAP_CPU;  in gaudi_init_cpu()
2832 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_cpu_queues()  local
2842 	if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)  in gaudi_init_cpu_queues()
2867 	if (gaudi->multi_msi_mode)  in gaudi_init_cpu_queues()
2889 	gaudi->hw_cap_initialized |= HW_CAP_CPU_Q;  in gaudi_init_cpu_queues()
3014 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hw_fini()  local
3080 	gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |  in gaudi_hw_fini()
3089 	memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat));  in gaudi_hw_fini()
3125 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_ring_doorbell()  local
3188 		if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)  in gaudi_ring_doorbell()
3411 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_get_int_queue_base()  local
3420 	q = &gaudi->internal_qmans[queue_id];  in gaudi_get_int_queue_base()
3430 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_send_cpu_message()  local
3432 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) {  in gaudi_send_cpu_message()
3524 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_test_cpu_queue()  local
3530 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_test_cpu_queue()
4277 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_cs_parser()  local
4282 	if (gaudi->hw_cap_initialized & HW_CAP_MMU)  in gaudi_cs_parser()
4293 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_add_end_of_cb_packets()  local
4316 	if (!gaudi->multi_msi_mode)  in gaudi_add_end_of_cb_packets()
4513 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_clear_pgt_range()  local
4517 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))  in gaudi_mmu_clear_pgt_range()
4531 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_read32()  local
4537 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_read32()
4578 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_write32()  local
4584 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_write32()
4625 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_read64()  local
4631 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_read64()
4676 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_write64()  local
4682 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_write64()
4725 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_read_pte()  local
4731 			(addr - gaudi->hbm_bar_cur_addr));  in gaudi_read_pte()
4736 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_write_pte()  local
4742 			(addr - gaudi->hbm_bar_cur_addr));  in gaudi_write_pte()
4754 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_prepare()  local
4756 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))  in gaudi_mmu_prepare()
4764 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_mmu_prepare()
4915 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_mmu_prepare()
5192 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_print_mmu_error_info()  local
5196 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))  in gaudi_print_mmu_error_info()
5246 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_extract_ecc_info()  local
5263 		mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_extract_ecc_info()
5308 		mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_extract_ecc_info()
5516 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_soft_reset_late_init()  local
5521 	return hl_fw_unmask_irq_arr(hdev, gaudi->events, sizeof(gaudi->events));  in gaudi_soft_reset_late_init()
5626 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_tpc_read_interrupts()  local
5635 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_tpc_read_interrupts()
5658 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_tpc_read_interrupts()
5711 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_handle_eqe()  local
5718 	gaudi->events_stat[event_type]++;  in gaudi_handle_eqe()
5719 	gaudi->events_stat_aggregate[event_type]++;  in gaudi_handle_eqe()
5916 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_get_events_stat()  local
5919 		*size = (u32) sizeof(gaudi->events_stat_aggregate);  in gaudi_get_events_stat()
5920 		return gaudi->events_stat_aggregate;  in gaudi_get_events_stat()
5923 	*size = (u32) sizeof(gaudi->events_stat);  in gaudi_get_events_stat()
5924 	return gaudi->events_stat;  in gaudi_get_events_stat()
5930 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_invalidate_cache()  local
5934 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||  in gaudi_mmu_invalidate_cache()
5947 	WREG32(mmSTLB_CACHE_INV, gaudi->mmu_cache_inv_pi++);  in gaudi_mmu_invalidate_cache()
5974 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_invalidate_cache_range()  local
5980 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||  in gaudi_mmu_invalidate_cache_range()
6059 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_send_heartbeat()  local
6061 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_send_heartbeat()
6069 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_cpucp_info_get()  local
6073 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_cpucp_info_get()
6099 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_is_device_idle()  local
6107 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_is_device_idle()
6199 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_is_device_idle()
6205 	__acquires(&gaudi->hw_queues_lock)  in gaudi_hw_queues_lock()
6207 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hw_queues_lock()  local
6209 	spin_lock(&gaudi->hw_queues_lock);  in gaudi_hw_queues_lock()
6213 	__releases(&gaudi->hw_queues_lock)  in gaudi_hw_queues_unlock()
6215 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hw_queues_unlock()  local
6217 	spin_unlock(&gaudi->hw_queues_lock);  in gaudi_hw_queues_unlock()
6228 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_get_eeprom_data()  local
6230 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_get_eeprom_data()
6243 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_run_tpc_kernel()  local
6255 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()
6299 		mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()
6324 		mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()
6337 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()