Lines Matching +full:max +full:- +full:outbound +full:- +full:regions
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2020-2022 HabanaLabs, Ltd.
45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
47 * for MAX faulty TPCs which reflects the cluster binning requirements
127 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0)
128 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0)
132 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
135 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
165 /* HW scrambles only bits 0-25 */
795 "FENCE 0 inc over max value and clipped",
796 "FENCE 1 inc over max value and clipped",
797 "FENCE 2 inc over max value and clipped",
798 "FENCE 3 inc over max value and clipped",
816 "FENCE 0 inc over max value and clipped",
817 "FENCE 1 inc over max value and clipped",
818 "FENCE 2 inc over max value and clipped",
819 "FENCE 3 inc over max value and clipped",
2015 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)-
2038 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
2039 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
2066 {"mmu rei0", -1}, /* no clear register bit */
2067 {"mmu rei1", -1}, /* no clear register bit */
2068 {"stlb rei0", -1}, /* no clear register bit */
2069 {"stlb rei1", -1}, /* no clear register bit */
2138 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_iterate_tpcs()
2143 ctx->rc = 0; in gaudi2_iterate_tpcs()
2149 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) in gaudi2_iterate_tpcs()
2154 ctx->fn(hdev, dcore, inst, offset, ctx); in gaudi2_iterate_tpcs()
2155 if (ctx->rc) { in gaudi2_iterate_tpcs()
2156 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", in gaudi2_iterate_tpcs()
2163 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) in gaudi2_iterate_tpcs()
2167 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); in gaudi2_iterate_tpcs()
2168 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); in gaudi2_iterate_tpcs()
2169 if (ctx->rc) in gaudi2_iterate_tpcs()
2170 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); in gaudi2_iterate_tpcs()
2183 struct asic_fixed_properties *prop = &hdev->asic_prop; in set_number_of_functional_hbms()
2184 u8 faulty_hbms = hweight64(hdev->dram_binning); in set_number_of_functional_hbms()
2188 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); in set_number_of_functional_hbms()
2189 prop->num_functional_hbms = GAUDI2_HBM_NUM; in set_number_of_functional_hbms()
2200 dev_err(hdev->dev, in set_number_of_functional_hbms()
2201 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", in set_number_of_functional_hbms()
2202 MAX_FAULTY_HBMS, hdev->dram_binning); in set_number_of_functional_hbms()
2203 return -EINVAL; in set_number_of_functional_hbms()
2208 * GAUDI2_HBM_NUM - 1. in set_number_of_functional_hbms()
2210 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; in set_number_of_functional_hbms()
2216 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_properties()
2222 return -EINVAL; in gaudi2_set_dram_properties()
2229 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; in gaudi2_set_dram_properties()
2230 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; in gaudi2_set_dram_properties()
2231 prop->device_mem_alloc_default_page_size = prop->dram_page_size; in gaudi2_set_dram_properties()
2232 prop->dram_size = prop->num_functional_hbms * SZ_16G; in gaudi2_set_dram_properties()
2233 prop->dram_base_address = DRAM_PHYS_BASE; in gaudi2_set_dram_properties()
2234 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in gaudi2_set_dram_properties()
2235 prop->dram_supports_virtual_memory = true; in gaudi2_set_dram_properties()
2237 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; in gaudi2_set_dram_properties()
2238 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; in gaudi2_set_dram_properties()
2239 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; in gaudi2_set_dram_properties()
2240 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; in gaudi2_set_dram_properties()
2248 * 1. partition the virtual address space to DRAM-page (whole) pages. in gaudi2_set_dram_properties()
2259 prop->dmmu.start_addr = prop->dram_base_address + in gaudi2_set_dram_properties()
2260 (prop->dram_page_size * in gaudi2_set_dram_properties()
2261 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); in gaudi2_set_dram_properties()
2263 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * in gaudi2_set_dram_properties()
2264 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); in gaudi2_set_dram_properties()
2271 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_fixed_properties()
2276 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; in gaudi2_set_fixed_properties()
2277 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), in gaudi2_set_fixed_properties()
2280 if (!prop->hw_queues_props) in gaudi2_set_fixed_properties()
2281 return -ENOMEM; in gaudi2_set_fixed_properties()
2283 q_props = prop->hw_queues_props; in gaudi2_set_fixed_properties()
2303 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; in gaudi2_set_fixed_properties()
2304 prop->cfg_base_address = CFG_BASE; in gaudi2_set_fixed_properties()
2305 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2306 prop->host_base_address = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2307 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; in gaudi2_set_fixed_properties()
2308 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; in gaudi2_set_fixed_properties()
2309 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2310 prop->user_dec_intr_count = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2311 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; in gaudi2_set_fixed_properties()
2312 prop->completion_mode = HL_COMPLETION_MODE_CS; in gaudi2_set_fixed_properties()
2313 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; in gaudi2_set_fixed_properties()
2314 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; in gaudi2_set_fixed_properties()
2316 prop->sram_base_address = SRAM_BASE_ADDR; in gaudi2_set_fixed_properties()
2317 prop->sram_size = SRAM_SIZE; in gaudi2_set_fixed_properties()
2318 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in gaudi2_set_fixed_properties()
2319 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; in gaudi2_set_fixed_properties()
2321 prop->hints_range_reservation = true; in gaudi2_set_fixed_properties()
2323 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; in gaudi2_set_fixed_properties()
2325 if (hdev->pldm) in gaudi2_set_fixed_properties()
2326 prop->mmu_pgt_size = 0x800000; /* 8MB */ in gaudi2_set_fixed_properties()
2328 prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE; in gaudi2_set_fixed_properties()
2330 prop->mmu_pte_size = HL_PTE_SIZE; in gaudi2_set_fixed_properties()
2331 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; in gaudi2_set_fixed_properties()
2332 prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; in gaudi2_set_fixed_properties()
2334 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; in gaudi2_set_fixed_properties()
2335 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; in gaudi2_set_fixed_properties()
2336 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; in gaudi2_set_fixed_properties()
2337 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; in gaudi2_set_fixed_properties()
2338 prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT; in gaudi2_set_fixed_properties()
2339 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; in gaudi2_set_fixed_properties()
2340 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; in gaudi2_set_fixed_properties()
2341 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; in gaudi2_set_fixed_properties()
2342 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; in gaudi2_set_fixed_properties()
2343 prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK; in gaudi2_set_fixed_properties()
2344 prop->dmmu.page_size = PAGE_SIZE_1GB; in gaudi2_set_fixed_properties()
2345 prop->dmmu.num_hops = MMU_ARCH_6_HOPS; in gaudi2_set_fixed_properties()
2346 prop->dmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2347 prop->dmmu.host_resident = 1; in gaudi2_set_fixed_properties()
2348 prop->dmmu.hop_table_size = prop->mmu_hop_table_size; in gaudi2_set_fixed_properties()
2349 prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; in gaudi2_set_fixed_properties()
2357 prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G; in gaudi2_set_fixed_properties()
2359 hdev->pmmu_huge_range = true; in gaudi2_set_fixed_properties()
2360 prop->pmmu.host_resident = 1; in gaudi2_set_fixed_properties()
2361 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; in gaudi2_set_fixed_properties()
2362 prop->pmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2363 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; in gaudi2_set_fixed_properties()
2364 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; in gaudi2_set_fixed_properties()
2366 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; in gaudi2_set_fixed_properties()
2367 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; in gaudi2_set_fixed_properties()
2368 prop->hints_host_hpage_reserved_va_range.start_addr = in gaudi2_set_fixed_properties()
2370 prop->hints_host_hpage_reserved_va_range.end_addr = in gaudi2_set_fixed_properties()
2374 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; in gaudi2_set_fixed_properties()
2375 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; in gaudi2_set_fixed_properties()
2376 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; in gaudi2_set_fixed_properties()
2377 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; in gaudi2_set_fixed_properties()
2378 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; in gaudi2_set_fixed_properties()
2379 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; in gaudi2_set_fixed_properties()
2380 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; in gaudi2_set_fixed_properties()
2381 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; in gaudi2_set_fixed_properties()
2382 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; in gaudi2_set_fixed_properties()
2383 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; in gaudi2_set_fixed_properties()
2384 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; in gaudi2_set_fixed_properties()
2385 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; in gaudi2_set_fixed_properties()
2386 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2387 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2388 prop->pmmu.page_size = PAGE_SIZE_64KB; in gaudi2_set_fixed_properties()
2391 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2392 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; in gaudi2_set_fixed_properties()
2393 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2394 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2396 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; in gaudi2_set_fixed_properties()
2397 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; in gaudi2_set_fixed_properties()
2398 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; in gaudi2_set_fixed_properties()
2399 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; in gaudi2_set_fixed_properties()
2400 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; in gaudi2_set_fixed_properties()
2401 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; in gaudi2_set_fixed_properties()
2402 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; in gaudi2_set_fixed_properties()
2403 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; in gaudi2_set_fixed_properties()
2404 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; in gaudi2_set_fixed_properties()
2405 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; in gaudi2_set_fixed_properties()
2406 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; in gaudi2_set_fixed_properties()
2407 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; in gaudi2_set_fixed_properties()
2408 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2409 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2410 prop->pmmu.page_size = PAGE_SIZE_4KB; in gaudi2_set_fixed_properties()
2413 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2414 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in gaudi2_set_fixed_properties()
2415 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2416 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2419 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; in gaudi2_set_fixed_properties()
2420 prop->num_engine_cores = CPU_ID_MAX; in gaudi2_set_fixed_properties()
2421 prop->cfg_size = CFG_SIZE; in gaudi2_set_fixed_properties()
2422 prop->max_asid = MAX_ASID; in gaudi2_set_fixed_properties()
2423 prop->num_of_events = GAUDI2_EVENT_SIZE; in gaudi2_set_fixed_properties()
2425 prop->supports_engine_modes = true; in gaudi2_set_fixed_properties()
2427 prop->dc_power_default = DC_POWER_DEFAULT; in gaudi2_set_fixed_properties()
2429 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; in gaudi2_set_fixed_properties()
2430 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; in gaudi2_set_fixed_properties()
2431 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; in gaudi2_set_fixed_properties()
2432 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in gaudi2_set_fixed_properties()
2434 strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); in gaudi2_set_fixed_properties()
2436 prop->mme_master_slave_mode = 1; in gaudi2_set_fixed_properties()
2438 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + in gaudi2_set_fixed_properties()
2441 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + in gaudi2_set_fixed_properties()
2444 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; in gaudi2_set_fixed_properties()
2445 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; in gaudi2_set_fixed_properties()
2446 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; in gaudi2_set_fixed_properties()
2448 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2450 prop->fw_cpu_boot_dev_sts0_valid = false; in gaudi2_set_fixed_properties()
2451 prop->fw_cpu_boot_dev_sts1_valid = false; in gaudi2_set_fixed_properties()
2452 prop->hard_reset_done_by_fw = false; in gaudi2_set_fixed_properties()
2453 prop->gic_interrupts_enable = true; in gaudi2_set_fixed_properties()
2455 prop->server_type = HL_SERVER_TYPE_UNKNOWN; in gaudi2_set_fixed_properties()
2457 prop->max_dec = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2459 prop->clk_pll_index = HL_GAUDI2_MME_PLL; in gaudi2_set_fixed_properties()
2461 prop->dma_mask = 64; in gaudi2_set_fixed_properties()
2463 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; in gaudi2_set_fixed_properties()
2478 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); in gaudi2_pci_bars_map()
2485 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_hbm_bar_base()
2490 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) in gaudi2_set_hbm_bar_base()
2493 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_set_hbm_bar_base()
2496 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_set_hbm_bar_base()
2505 old_addr = gaudi2->dram_bar_cur_addr; in gaudi2_set_hbm_bar_base()
2506 gaudi2->dram_bar_cur_addr = addr; in gaudi2_set_hbm_bar_base()
2519 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_init_iatu()
2522 /* Temporary inbound Region 0 - Bar 0 - Point to CFG in gaudi2_init_iatu()
2529 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; in gaudi2_init_iatu()
2538 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; in gaudi2_init_iatu()
2540 /* Inbound Region 0 - Bar 0 - Point to CFG */ in gaudi2_init_iatu()
2550 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ in gaudi2_init_iatu()
2560 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_init_iatu()
2568 /* Outbound Region 0 - Point to Host */ in gaudi2_init_iatu()
2583 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_tpc_binning_init_prop()
2589 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { in gaudi2_tpc_binning_init_prop()
2590 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", in gaudi2_tpc_binning_init_prop()
2592 hdev->tpc_binning); in gaudi2_tpc_binning_init_prop()
2593 return -EINVAL; in gaudi2_tpc_binning_init_prop()
2596 prop->tpc_binning_mask = hdev->tpc_binning; in gaudi2_tpc_binning_init_prop()
2597 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; in gaudi2_tpc_binning_init_prop()
2604 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_tpc_binning_masks()
2605 struct hw_queue_properties *q_props = prop->hw_queues_props; in gaudi2_set_tpc_binning_masks()
2614 tpc_binning_mask = prop->tpc_binning_mask; in gaudi2_set_tpc_binning_masks()
2634 * Coverity complains about possible out-of-bound access in in gaudi2_set_tpc_binning_masks()
2638 dev_err(hdev->dev, in gaudi2_set_tpc_binning_masks()
2641 return -EINVAL; in gaudi2_set_tpc_binning_masks()
2646 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); in gaudi2_set_tpc_binning_masks()
2662 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dec_binning_masks()
2665 num_faulty = hweight32(hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2672 …dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0… in gaudi2_set_dec_binning_masks()
2673 hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2674 return -EINVAL; in gaudi2_set_dec_binning_masks()
2677 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); in gaudi2_set_dec_binning_masks()
2679 if (prop->decoder_binning_mask) in gaudi2_set_dec_binning_masks()
2680 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); in gaudi2_set_dec_binning_masks()
2682 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; in gaudi2_set_dec_binning_masks()
2689 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_binning_masks()
2692 if (!hdev->dram_binning) { in gaudi2_set_dram_binning_masks()
2693 prop->dram_binning_mask = 0; in gaudi2_set_dram_binning_masks()
2694 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; in gaudi2_set_dram_binning_masks()
2699 prop->faulty_dram_cluster_map |= hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2700 prop->dram_binning_mask = hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2701 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); in gaudi2_set_dram_binning_masks()
2706 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_edma_binning_masks()
2710 num_faulty = hweight32(hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2717 dev_err(hdev->dev, in gaudi2_set_edma_binning_masks()
2718 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", in gaudi2_set_edma_binning_masks()
2719 hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2720 return -EINVAL; in gaudi2_set_edma_binning_masks()
2723 if (!hdev->edma_binning) { in gaudi2_set_edma_binning_masks()
2724 prop->edma_binning_mask = 0; in gaudi2_set_edma_binning_masks()
2725 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; in gaudi2_set_edma_binning_masks()
2729 seq = __ffs((unsigned long)hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2732 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); in gaudi2_set_edma_binning_masks()
2733 prop->edma_binning_mask = hdev->edma_binning; in gaudi2_set_edma_binning_masks()
2734 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); in gaudi2_set_edma_binning_masks()
2737 q_props = prop->hw_queues_props; in gaudi2_set_edma_binning_masks()
2748 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_xbar_edge_enable_mask()
2753 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2768 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", in gaudi2_set_xbar_edge_enable_mask()
2770 return -EINVAL; in gaudi2_set_xbar_edge_enable_mask()
2776 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); in gaudi2_set_xbar_edge_enable_mask()
2777 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2789 * If more than single cluster is faulty- the chip is unusable in gaudi2_set_cluster_binning_masks_common()
2791 hdev->asic_prop.faulty_dram_cluster_map = 0; in gaudi2_set_cluster_binning_masks_common()
2805 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; in gaudi2_set_cluster_binning_masks_common()
2812 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_cluster_binning_masks()
2815 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); in gaudi2_set_cluster_binning_masks()
2820 if (prop->faulty_dram_cluster_map) { in gaudi2_set_cluster_binning_masks()
2821 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); in gaudi2_set_cluster_binning_masks()
2823 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; in gaudi2_set_cluster_binning_masks()
2850 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cpucp_info_get()
2851 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_cpucp_info_get()
2856 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_cpucp_info_get()
2862 if (hdev->reset_info.in_compute_reset) in gaudi2_cpucp_info_get()
2870 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); in gaudi2_cpucp_info_get()
2874 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && in gaudi2_cpucp_info_get()
2876 dev_err(hdev->dev, in gaudi2_cpucp_info_get()
2878 dram_size, prop->dram_size); in gaudi2_cpucp_info_get()
2879 dram_size = prop->dram_size; in gaudi2_cpucp_info_get()
2882 prop->dram_size = dram_size; in gaudi2_cpucp_info_get()
2883 prop->dram_end_address = prop->dram_base_address + dram_size; in gaudi2_cpucp_info_get()
2886 if (!strlen(prop->cpucp_info.card_name)) in gaudi2_cpucp_info_get()
2887 strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); in gaudi2_cpucp_info_get()
2890 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; in gaudi2_cpucp_info_get()
2891 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; in gaudi2_cpucp_info_get()
2892 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); in gaudi2_cpucp_info_get()
2893 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); in gaudi2_cpucp_info_get()
2895 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", in gaudi2_cpucp_info_get()
2896 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, in gaudi2_cpucp_info_get()
2897 hdev->decoder_binning); in gaudi2_cpucp_info_get()
2903 rc = hdev->asic_funcs->set_dram_properties(hdev); in gaudi2_cpucp_info_get()
2907 rc = hdev->asic_funcs->set_binning_masks(hdev); in gaudi2_cpucp_info_get()
2915 prop->max_power_default = (u64) max_power; in gaudi2_cpucp_info_get()
2922 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_fetch_psoc_frequency()
2926 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_fetch_psoc_frequency()
2933 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; in gaudi2_fetch_psoc_frequency()
2940 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_early_init()
2941 struct pci_dev *pdev = hdev->pdev; in gaudi2_early_init()
2953 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
2955 rc = -ENODEV; in gaudi2_early_init()
2961 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
2963 rc = -ENODEV; in gaudi2_early_init()
2967 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); in gaudi2_early_init()
2968 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); in gaudi2_early_init()
2973 if (hdev->pldm) in gaudi2_early_init()
2974 hdev->asic_prop.iatu_done_by_fw = false; in gaudi2_early_init()
2976 hdev->asic_prop.iatu_done_by_fw = true; in gaudi2_early_init()
2983 * version to determine whether we run with a security-enabled firmware in gaudi2_early_init()
2987 if (hdev->reset_on_preboot_fail) in gaudi2_early_init()
2989 hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
2994 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); in gaudi2_early_init()
2995 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
2997 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); in gaudi2_early_init()
3007 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_init()
3013 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_fini()
3041 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_arcs()
3042 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_arcs()
3062 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) in gaudi2_init_arcs()
3065 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & in gaudi2_init_arcs()
3066 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) in gaudi2_init_arcs()
3073 hdev->asic_prop.engine_core_interrupt_reg_addr = in gaudi2_init_arcs()
3074 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); in gaudi2_init_arcs()
3145 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_late_init()
3148 hdev->asic_prop.supports_advanced_cpucp_rc = true; in gaudi2_late_init()
3151 gaudi2->virt_msix_db_dma_addr); in gaudi2_late_init()
3153 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); in gaudi2_late_init()
3159 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); in gaudi2_late_init()
3167 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_late_init()
3188 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_dec_init()
3204 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_user_mapped_blocks_init()
3205 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_blocks_init()
3263 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3264 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3266 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = in gaudi2_user_mapped_blocks_init()
3269 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = in gaudi2_user_mapped_blocks_init()
3280 /* The device ARC works with 32-bits addresses, and because there is a single HW register in gaudi2_alloc_cpu_accessible_dma_mem()
3289 rc = -ENOMEM; in gaudi2_alloc_cpu_accessible_dma_mem()
3293 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; in gaudi2_alloc_cpu_accessible_dma_mem()
3299 dev_err(hdev->dev, in gaudi2_alloc_cpu_accessible_dma_mem()
3301 rc = -EFAULT; in gaudi2_alloc_cpu_accessible_dma_mem()
3305 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3306 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3318 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_pci_memory_regions()
3322 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in gaudi2_set_pci_memory_regions()
3323 region->region_base = CFG_BASE; in gaudi2_set_pci_memory_regions()
3324 region->region_size = CFG_SIZE; in gaudi2_set_pci_memory_regions()
3325 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3326 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3327 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3328 region->used = 1; in gaudi2_set_pci_memory_regions()
3331 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in gaudi2_set_pci_memory_regions()
3332 region->region_base = SRAM_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3333 region->region_size = SRAM_SIZE; in gaudi2_set_pci_memory_regions()
3334 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; in gaudi2_set_pci_memory_regions()
3335 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3336 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3337 region->used = 1; in gaudi2_set_pci_memory_regions()
3340 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in gaudi2_set_pci_memory_regions()
3341 region->region_base = DRAM_PHYS_BASE; in gaudi2_set_pci_memory_regions()
3342 region->region_size = hdev->asic_prop.dram_size; in gaudi2_set_pci_memory_regions()
3343 region->offset_in_bar = 0; in gaudi2_set_pci_memory_regions()
3344 region->bar_size = prop->dram_pci_bar_size; in gaudi2_set_pci_memory_regions()
3345 region->bar_id = DRAM_BAR_ID; in gaudi2_set_pci_memory_regions()
3346 region->used = 1; in gaudi2_set_pci_memory_regions()
3351 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_user_interrupt_setup()
3355 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); in gaudi2_user_interrupt_setup()
3358 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, in gaudi2_user_interrupt_setup()
3362 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, in gaudi2_user_interrupt_setup()
3366 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, in gaudi2_user_interrupt_setup()
3380 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, in gaudi2_user_interrupt_setup()
3383 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) in gaudi2_user_interrupt_setup()
3384 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); in gaudi2_user_interrupt_setup()
3396 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_free()
3398 &prop->skip_special_blocks_cfg; in gaudi2_special_blocks_free()
3400 kfree(prop->special_blocks); in gaudi2_special_blocks_free()
3401 kfree(skip_special_blocks_cfg->block_types); in gaudi2_special_blocks_free()
3402 kfree(skip_special_blocks_cfg->block_ranges); in gaudi2_special_blocks_free()
3419 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_config()
3423 prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE; in gaudi2_special_blocks_config()
3424 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); in gaudi2_special_blocks_config()
3425 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, in gaudi2_special_blocks_config()
3426 sizeof(*prop->special_blocks), GFP_KERNEL); in gaudi2_special_blocks_config()
3427 if (!prop->special_blocks) in gaudi2_special_blocks_config()
3428 return -ENOMEM; in gaudi2_special_blocks_config()
3430 for (i = 0 ; i < prop->num_of_special_blocks ; i++) in gaudi2_special_blocks_config()
3431 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], in gaudi2_special_blocks_config()
3432 sizeof(*prop->special_blocks)); in gaudi2_special_blocks_config()
3435 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); in gaudi2_special_blocks_config()
3436 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; in gaudi2_special_blocks_config()
3439 prop->skip_special_blocks_cfg.block_types = in gaudi2_special_blocks_config()
3442 if (!prop->skip_special_blocks_cfg.block_types) { in gaudi2_special_blocks_config()
3443 rc = -ENOMEM; in gaudi2_special_blocks_config()
3447 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, in gaudi2_special_blocks_config()
3450 prop->skip_special_blocks_cfg.block_types_len = in gaudi2_special_blocks_config()
3455 prop->skip_special_blocks_cfg.block_ranges = in gaudi2_special_blocks_config()
3458 if (!prop->skip_special_blocks_cfg.block_ranges) { in gaudi2_special_blocks_config()
3459 rc = -ENOMEM; in gaudi2_special_blocks_config()
3464 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], in gaudi2_special_blocks_config()
3468 prop->skip_special_blocks_cfg.block_ranges_len = in gaudi2_special_blocks_config()
3475 kfree(prop->skip_special_blocks_cfg.block_types); in gaudi2_special_blocks_config()
3477 kfree(prop->special_blocks); in gaudi2_special_blocks_config()
3489 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_free()
3490 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_free()
3494 /* bail-out if this is an allocation failure point */ in gaudi2_test_queues_msgs_free()
3505 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_alloc()
3506 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_alloc()
3509 /* allocate a message-short buf for each Q we intend to test */ in gaudi2_test_queues_msgs_alloc()
3515 dev_err(hdev->dev, in gaudi2_test_queues_msgs_alloc()
3517 rc = -ENOMEM; in gaudi2_test_queues_msgs_alloc()
3531 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_init()
3538 return -ENOMEM; in gaudi2_sw_init()
3544 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { in gaudi2_sw_init()
3545 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", in gaudi2_sw_init()
3547 rc = -EINVAL; in gaudi2_sw_init()
3551 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; in gaudi2_sw_init()
3555 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); in gaudi2_sw_init()
3557 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; in gaudi2_sw_init()
3559 hdev->asic_specific = gaudi2; in gaudi2_sw_init()
3562 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped in gaudi2_sw_init()
3565 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, in gaudi2_sw_init()
3567 if (!hdev->dma_pool) { in gaudi2_sw_init()
3568 dev_err(hdev->dev, "failed to create DMA pool\n"); in gaudi2_sw_init()
3569 rc = -ENOMEM; in gaudi2_sw_init()
3577 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in gaudi2_sw_init()
3578 if (!hdev->cpu_accessible_dma_pool) { in gaudi2_sw_init()
3579 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); in gaudi2_sw_init()
3580 rc = -ENOMEM; in gaudi2_sw_init()
3584 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3585 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in gaudi2_sw_init()
3587 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); in gaudi2_sw_init()
3588 rc = -EFAULT; in gaudi2_sw_init()
3592 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, in gaudi2_sw_init()
3593 &gaudi2->virt_msix_db_dma_addr); in gaudi2_sw_init()
3594 if (!gaudi2->virt_msix_db_cpu_addr) { in gaudi2_sw_init()
3595 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); in gaudi2_sw_init()
3596 rc = -ENOMEM; in gaudi2_sw_init()
3600 spin_lock_init(&gaudi2->hw_queues_lock); in gaudi2_sw_init()
3602 gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE, in gaudi2_sw_init()
3603 &gaudi2->scratchpad_bus_address, in gaudi2_sw_init()
3605 if (!gaudi2->scratchpad_kernel_address) { in gaudi2_sw_init()
3606 rc = -ENOMEM; in gaudi2_sw_init()
3615 hdev->supports_coresight = true; in gaudi2_sw_init()
3616 hdev->supports_sync_stream = true; in gaudi2_sw_init()
3617 hdev->supports_cb_mapping = true; in gaudi2_sw_init()
3618 hdev->supports_wait_for_multi_cs = false; in gaudi2_sw_init()
3620 prop->supports_compute_reset = true; in gaudi2_sw_init()
3624 hdev->event_queue.check_eqe_index = false; in gaudi2_sw_init()
3626 hdev->event_queue.check_eqe_index = true; in gaudi2_sw_init()
3628 hdev->asic_funcs->set_pci_memory_regions(hdev); in gaudi2_sw_init()
3643 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, in gaudi2_sw_init()
3644 gaudi2->scratchpad_bus_address); in gaudi2_sw_init()
3646 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_init()
3648 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_init()
3650 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3651 hdev->cpu_accessible_dma_address); in gaudi2_sw_init()
3653 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_init()
3661 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_fini()
3662 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sw_fini()
3668 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_fini()
3670 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_fini()
3672 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_fini()
3673 hdev->cpu_accessible_dma_address); in gaudi2_sw_fini()
3675 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, in gaudi2_sw_fini()
3676 gaudi2->scratchpad_bus_address); in gaudi2_sw_fini()
3678 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_fini()
3708 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3725 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; in gaudi2_clear_qm_fence_counters_common()
3748 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dma_qmans()
3751 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_stop_dma_qmans()
3759 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_stop_dma_qmans()
3767 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_stop_dma_qmans()
3781 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_mme_qmans()
3784 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_stop_mme_qmans()
3787 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) in gaudi2_stop_mme_qmans()
3796 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_tpc_qmans()
3800 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_stop_tpc_qmans()
3804 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_stop_tpc_qmans()
3814 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_rot_qmans()
3818 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_stop_rot_qmans()
3822 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_stop_rot_qmans()
3832 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_nic_qmans()
3836 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_stop_nic_qmans()
3842 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_stop_nic_qmans()
3860 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dma_stall()
3863 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_dma_stall()
3870 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_dma_stall()
3878 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_dma_stall()
3892 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mme_stall()
3895 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; in gaudi2_mme_stall()
3898 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_mme_stall()
3904 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_tpc_stall()
3908 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_tpc_stall()
3912 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_tpc_stall()
3922 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_rotator_stall()
3926 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_rotator_stall()
3934 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_rotator_stall()
3948 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_dma_qmans()
3951 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_disable_dma_qmans()
3958 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_disable_dma_qmans()
3966 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_disable_dma_qmans()
3980 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_mme_qmans()
3983 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_disable_mme_qmans()
3986 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_disable_mme_qmans()
3992 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_tpc_qmans()
3996 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_disable_tpc_qmans()
4000 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_disable_tpc_qmans()
4010 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_rot_qmans()
4014 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_disable_rot_qmans()
4018 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_disable_rot_qmans()
4028 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_nic_qmans()
4032 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_disable_nic_qmans()
4038 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_disable_nic_qmans()
4051 /* Zero the lower/upper parts of the 64-bit counter */ in gaudi2_enable_timestamp()
4073 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; in gaudi2_irq_name()
4091 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_disable_msix()
4092 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_disable_msix()
4094 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_disable_msix()
4102 (void *) &hdev->user_interrupt[dec->core_id])); in gaudi2_dec_disable_msix()
4115 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_enable_msix()
4116 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_enable_msix()
4125 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_enable_msix()
4133 (void *) &hdev->user_interrupt[dec->core_id]); in gaudi2_dec_enable_msix()
4137 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_dec_enable_msix()
4151 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_enable_msix()
4152 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_msix()
4156 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) in gaudi2_enable_msix()
4159 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, in gaudi2_enable_msix()
4162 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", in gaudi2_enable_msix()
4167 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4168 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_enable_msix()
4171 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4175 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4177 &hdev->event_queue); in gaudi2_enable_msix()
4179 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4185 dev_err(hdev->dev, "Failed to enable decoder IRQ"); in gaudi2_enable_msix()
4189 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4192 gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), &hdev->tpc_interrupt); in gaudi2_enable_msix()
4194 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4198 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4201 &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4203 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4207 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; in gaudi2_enable_msix()
4208 user_irq_init_cnt < prop->user_interrupt_count; in gaudi2_enable_msix()
4211 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4214 gaudi2_irq_name(i), &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4217 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4222 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; in gaudi2_enable_msix()
4227 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; in gaudi2_enable_msix()
4230 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4231 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4233 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4234 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4236 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4237 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_enable_msix()
4241 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4245 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4249 pci_free_irq_vectors(hdev->pdev); in gaudi2_enable_msix()
4256 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sync_irqs()
4260 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_sync_irqs()
4264 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); in gaudi2_sync_irqs()
4267 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4271 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); in gaudi2_sync_irqs()
4272 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); in gaudi2_sync_irqs()
4274 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; in gaudi2_sync_irqs()
4276 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4280 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); in gaudi2_sync_irqs()
4285 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_disable_msix()
4286 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_msix()
4290 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_disable_msix()
4295 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_disable_msix()
4296 free_irq(irq, &hdev->event_queue); in gaudi2_disable_msix()
4300 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_disable_msix()
4301 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_disable_msix()
4303 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_disable_msix()
4304 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_disable_msix()
4306 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; in gaudi2_disable_msix()
4307 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { in gaudi2_disable_msix()
4309 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_disable_msix()
4310 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_disable_msix()
4313 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_disable_msix()
4314 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_disable_msix()
4317 pci_free_irq_vectors(hdev->pdev); in gaudi2_disable_msix()
4319 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; in gaudi2_disable_msix()
4329 if (hdev->pldm) in gaudi2_stop_dcore_dec()
4336 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_dcore_dec()
4356 dev_err(hdev->dev, in gaudi2_stop_dcore_dec()
4369 if (hdev->pldm) in gaudi2_stop_pcie_dec()
4376 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_pcie_dec()
4396 dev_err(hdev->dev, in gaudi2_stop_pcie_dec()
4404 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dec()
4407 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) in gaudi2_stop_dec()
4444 if (hdev->pldm) in gaudi2_verify_arc_running_mode()
4468 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_reset_arcs()
4481 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_nic_qmans_manual_flush()
4485 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_nic_qmans_manual_flush()
4491 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_nic_qmans_manual_flush()
4513 dev_err(hdev->dev, "failed to %s arc: %d\n", in gaudi2_set_engine_cores()
4516 return -1; in gaudi2_set_engine_cores()
4526 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_tpc_engine_mode()
4529 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_set_tpc_engine_mode()
4533 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) in gaudi2_set_tpc_engine_mode()
4553 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_mme_engine_mode()
4557 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) in gaudi2_set_mme_engine_mode()
4571 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_edma_engine_mode()
4574 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_set_edma_engine_mode()
4578 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) in gaudi2_set_edma_engine_mode()
4631 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); in gaudi2_set_engine_modes()
4632 return -EINVAL; in gaudi2_set_engine_modes()
4652 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); in gaudi2_set_engines()
4653 return -EINVAL; in gaudi2_set_engines()
4661 if (hdev->pldm) in gaudi2_halt_engines()
4711 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; in gaudi2_init_firmware_preload_params()
4713 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in gaudi2_init_firmware_preload_params()
4714 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; in gaudi2_init_firmware_preload_params()
4715 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; in gaudi2_init_firmware_preload_params()
4716 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; in gaudi2_init_firmware_preload_params()
4717 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; in gaudi2_init_firmware_preload_params()
4718 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_preload_params()
4723 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in gaudi2_init_firmware_loader()
4728 fw_loader->fw_comp_loaded = FW_TYPE_NONE; in gaudi2_init_firmware_loader()
4729 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; in gaudi2_init_firmware_loader()
4730 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; in gaudi2_init_firmware_loader()
4731 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4732 fw_loader->skip_bmc = false; in gaudi2_init_firmware_loader()
4733 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; in gaudi2_init_firmware_loader()
4734 fw_loader->dram_bar_id = DRAM_BAR_ID; in gaudi2_init_firmware_loader()
4735 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4739 * hard-coded). in later stages of the protocol those values will be in gaudi2_init_firmware_loader()
4741 * will always be up-to-date in gaudi2_init_firmware_loader()
4743 dynamic_loader = &hdev->fw_loader.dynamic_loader; in gaudi2_init_firmware_loader()
4744 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; in gaudi2_init_firmware_loader()
4745 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); in gaudi2_init_firmware_loader()
4746 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); in gaudi2_init_firmware_loader()
4747 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4752 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu()
4755 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in gaudi2_init_cpu()
4758 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) in gaudi2_init_cpu()
4765 gaudi2->hw_cap_initialized |= HW_CAP_CPU; in gaudi2_init_cpu()
4772 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_init_cpu_queues()
4773 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_cpu_queues()
4774 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu_queues()
4780 if (!hdev->cpu_queues_enable) in gaudi2_init_cpu_queues()
4783 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_init_cpu_queues()
4786 eq = &hdev->event_queue; in gaudi2_init_cpu_queues()
4788 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_cpu_queues()
4790 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4791 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4793 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4794 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4796 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4797 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4812 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_init_cpu_queues()
4824 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); in gaudi2_init_cpu_queues()
4825 return -EIO; in gaudi2_init_cpu_queues()
4829 if (prop->fw_cpu_boot_dev_sts0_valid) in gaudi2_init_cpu_queues()
4830 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); in gaudi2_init_cpu_queues()
4832 if (prop->fw_cpu_boot_dev_sts1_valid) in gaudi2_init_cpu_queues()
4833 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); in gaudi2_init_cpu_queues()
4835 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; in gaudi2_init_cpu_queues()
4846 q = &hdev->kernel_queues[queue_id_base + pq_id]; in gaudi2_init_qman_pq()
4850 lower_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
4852 upper_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
4884 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_qman_pqc()
4895 lower_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
4897 upper_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
4913 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_get_dyn_sp_reg()
4926 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
4935 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
4944 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
4947 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
4950 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
4953 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); in gaudi2_get_dyn_sp_reg()
5002 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; in gaudi2_init_qman()
5023 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_dma_core()
5024 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); in gaudi2_init_dma_core()
5042 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_kdma()
5045 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) in gaudi2_init_kdma()
5052 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; in gaudi2_init_kdma()
5057 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_pdma()
5060 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) in gaudi2_init_pdma()
5075 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; in gaudi2_init_pdma()
5094 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_edma()
5095 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_edma()
5098 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) in gaudi2_init_edma()
5105 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_init_edma()
5110 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); in gaudi2_init_edma()
5116 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
5123 * write directly to the HBW host memory of the virtual MSI-X doorbell.
5128 * completion, by decrementing the sync object value and re-arming the monitor.
5134 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_arm_monitors_for_virt_msix_db()
5143 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) in gaudi2_arm_monitors_for_virt_msix_db()
5145 * 3. Re-arm the master monitor. in gaudi2_arm_monitors_for_virt_msix_db()
5157 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ in gaudi2_arm_monitors_for_virt_msix_db()
5162 /* 3rd monitor: Re-arm the master monitor */ in gaudi2_arm_monitors_for_virt_msix_db()
5180 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ in gaudi2_arm_monitors_for_virt_msix_db()
5186 addr = gaudi2->virt_msix_db_dma_addr; in gaudi2_arm_monitors_for_virt_msix_db()
5199 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_prepare_sm_for_virt_msix_db()
5203 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) in gaudi2_prepare_sm_for_virt_msix_db()
5220 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_sm()
5236 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ in gaudi2_init_sm()
5237 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5238 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5243 hdev->completion_queue[i].bus_address; in gaudi2_init_sm()
5257 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ in gaudi2_init_sm()
5263 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme_acc()
5279 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); in gaudi2_init_mme_acc()
5302 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); in gaudi2_init_dcore_mme()
5317 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme()
5320 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) in gaudi2_init_mme()
5326 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); in gaudi2_init_mme()
5347 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc_config()
5348 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; in gaudi2_init_tpc_config()
5352 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); in gaudi2_init_tpc_config()
5354 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) in gaudi2_init_tpc_config()
5363 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); in gaudi2_init_tpc_config()
5368 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc()
5372 if (!hdev->asic_prop.tpc_enabled_mask) in gaudi2_init_tpc()
5375 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) in gaudi2_init_tpc()
5389 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_rotator()
5398 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); in gaudi2_init_rotator()
5421 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_dec()
5425 if (!hdev->asic_prop.decoder_enabled_mask) in gaudi2_init_dec()
5428 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) in gaudi2_init_dec()
5435 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5445 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5450 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5458 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5468 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_update_asid_hop0_addr()
5487 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); in gaudi2_mmu_update_asid_hop0_addr()
5514 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : in gaudi2_mmu_invalidate_cache_status_poll()
5518 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { in gaudi2_mmu_invalidate_cache_status_poll()
5537 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) in gaudi2_mmu_invalidate_cache_status_poll()
5540 start_offset = inv_params->range_invalidation ? in gaudi2_mmu_invalidate_cache_status_poll()
5556 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_hmmu_enabled()
5561 if (gaudi2->hw_cap_initialized & hw_cap) in gaudi2_is_hmmu_enabled()
5581 if (inv_params->range_invalidation) { in gaudi2_mmu_invalidate_cache_trigger()
5588 u64 start = inv_params->start_va - 1; in gaudi2_mmu_invalidate_cache_trigger()
5599 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5602 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5608 inv_params->inv_start_val, inv_params->flags); in gaudi2_mmu_invalidate_cache_trigger()
5664 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache()
5668 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache()
5674 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache()
5691 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache_range()
5696 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache_range()
5705 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache_range()
5741 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_update_hop0_addr()
5743 u32 asid, max_asid = prop->max_asid; in gaudi2_mmu_update_hop0_addr()
5747 if (hdev->pldm) in gaudi2_mmu_update_hop0_addr()
5751 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; in gaudi2_mmu_update_hop0_addr()
5754 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); in gaudi2_mmu_update_hop0_addr()
5767 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_init_common()
5783 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); in gaudi2_mmu_init_common()
5800 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); in gaudi2_mmu_init_common()
5809 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_pci_mmu_init()
5813 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) in gaudi2_pci_mmu_init()
5852 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; in gaudi2_pci_mmu_init()
5860 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_dcore_hmmu_init()
5861 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dcore_hmmu_init()
5873 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) in gaudi2_dcore_hmmu_init()
5904 gaudi2->hw_cap_initialized |= hw_cap; in gaudi2_dcore_hmmu_init()
5940 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_init()
5957 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_hw_init()
5958 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; in gaudi2_hw_init()
5961 * Before pushing u-boot/linux to device, need to set the hbm bar to in gaudi2_hw_init()
5965 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); in gaudi2_hw_init()
5966 return -EIO; in gaudi2_hw_init()
5971 dev_err(hdev->dev, "failed to initialize CPU\n"); in gaudi2_hw_init()
5980 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); in gaudi2_hw_init()
5984 rc = gaudi2->cpucp_info_get(hdev); in gaudi2_hw_init()
5986 dev_err(hdev->dev, "Failed to get cpucp info\n"); in gaudi2_hw_init()
6029 * gaudi2_send_hard_reset_cmd - common function to handle reset
6038 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_send_hard_reset_cmd()
6040 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_hard_reset_cmd()
6043 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); in gaudi2_send_hard_reset_cmd()
6044 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); in gaudi2_send_hard_reset_cmd()
6059 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && in gaudi2_send_hard_reset_cmd()
6072 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), in gaudi2_send_hard_reset_cmd()
6082 * For the case in which we are working with Linux/Bootfit this is a hail-mary in gaudi2_send_hard_reset_cmd()
6096 if (hdev->asic_prop.hard_reset_done_by_fw) in gaudi2_send_hard_reset_cmd()
6104 * gaudi2_execute_hard_reset - execute hard reset by driver/FW
6112 if (hdev->asic_prop.hard_reset_done_by_fw) { in gaudi2_execute_hard_reset()
6143 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n", in gaudi2_get_soft_rst_done_indication()
6149 * gaudi2_execute_soft_reset - execute soft reset by driver/FW
6160 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_execute_soft_reset()
6166 if (dyn_regs->cpu_rst_status) in gaudi2_execute_soft_reset()
6167 WREG32(le32_to_cpu(dyn_regs->cpu_rst_status), CPU_RST_STATUS_NA); in gaudi2_execute_soft_reset()
6170 WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq), in gaudi2_execute_soft_reset()
6185 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, in gaudi2_execute_soft_reset()
6188 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, in gaudi2_execute_soft_reset()
6214 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); in gaudi2_poll_btm_indication()
6219 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_fini()
6224 if (hdev->pldm) { in gaudi2_hw_fini()
6239 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; in gaudi2_hw_fini()
6248 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && in gaudi2_hw_fini()
6249 !hdev->asic_prop.fw_security_enabled); in gaudi2_hw_fini()
6261 * - setting the dirty bit in gaudi2_hw_fini()
6264 * - dirty bit cleared in gaudi2_hw_fini()
6265 * - BTM indication cleared in gaudi2_hw_fini()
6266 * - preboot ready indication cleared in gaudi2_hw_fini()
6268 * - BTM indication will be set in gaudi2_hw_fini()
6269 * - BIST test performed and another reset triggered in gaudi2_hw_fini()
6280 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) in gaudi2_hw_fini()
6289 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); in gaudi2_hw_fini()
6290 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); in gaudi2_hw_fini()
6293 * Clear NIC capability mask in order for driver to re-configure in gaudi2_hw_fini()
6294 * NIC QMANs. NIC ports will not be re-configured during soft in gaudi2_hw_fini()
6297 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); in gaudi2_hw_fini()
6300 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6307 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); in gaudi2_hw_fini()
6309 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6323 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); in gaudi2_suspend()
6343 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); in gaudi2_mmap()
6345 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in gaudi2_mmap()
6349 rc = remap_pfn_range(vma, vma->vm_start, in gaudi2_mmap()
6351 size, vma->vm_page_prot); in gaudi2_mmap()
6353 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_mmap()
6362 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_queue_enabled()
6376 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6380 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6384 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6388 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6409 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6413 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6418 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6423 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6428 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6436 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); in gaudi2_is_queue_enabled()
6440 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); in gaudi2_is_queue_enabled()
6444 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6448 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); in gaudi2_is_queue_enabled()
6455 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); in gaudi2_is_queue_enabled()
6458 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); in gaudi2_is_queue_enabled()
6463 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); in gaudi2_is_queue_enabled()
6468 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_arc_enabled()
6473 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); in gaudi2_is_arc_enabled()
6476 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6479 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6488 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_clr_arc_id_cap()
6493 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); in gaudi2_clr_arc_id_cap()
6497 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6501 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6511 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_arc_id_cap()
6516 gaudi2->active_hw_arc |= BIT_ULL(arc_id); in gaudi2_set_arc_id_cap()
6520 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6524 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6534 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_ring_doorbell()
6558 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_ring_doorbell()
6575 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); in gaudi2_dma_alloc_coherent()
6581 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); in gaudi2_dma_free_coherent()
6587 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_cpu_message()
6589 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { in gaudi2_send_cpu_message()
6607 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in gaudi2_dma_pool_zalloc()
6612 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); in gaudi2_dma_pool_free()
6631 dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, dir); in gaudi2_dma_map_single()
6632 if (unlikely(dma_mapping_error(&hdev->pdev->dev, dma_addr))) in gaudi2_dma_map_single()
6641 dma_unmap_single(&hdev->pdev->dev, addr, len, dir); in gaudi2_dma_unmap_single()
6646 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in gaudi2_validate_cb_address()
6647 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_validate_cb_address()
6649 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { in gaudi2_validate_cb_address()
6650 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); in gaudi2_validate_cb_address()
6651 return -EINVAL; in gaudi2_validate_cb_address()
6656 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6657 parser->user_cb_size, in gaudi2_validate_cb_address()
6658 asic_prop->sram_user_base_address, in gaudi2_validate_cb_address()
6659 asic_prop->sram_end_address)) in gaudi2_validate_cb_address()
6662 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6663 parser->user_cb_size, in gaudi2_validate_cb_address()
6664 asic_prop->dram_user_base_address, in gaudi2_validate_cb_address()
6665 asic_prop->dram_end_address)) in gaudi2_validate_cb_address()
6668 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && in gaudi2_validate_cb_address()
6669 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6670 parser->user_cb_size, in gaudi2_validate_cb_address()
6671 asic_prop->dmmu.start_addr, in gaudi2_validate_cb_address()
6672 asic_prop->dmmu.end_addr)) in gaudi2_validate_cb_address()
6675 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { in gaudi2_validate_cb_address()
6676 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6677 parser->user_cb_size, in gaudi2_validate_cb_address()
6678 asic_prop->pmmu.start_addr, in gaudi2_validate_cb_address()
6679 asic_prop->pmmu.end_addr) || in gaudi2_validate_cb_address()
6681 (u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6682 parser->user_cb_size, in gaudi2_validate_cb_address()
6683 asic_prop->pmmu_huge.start_addr, in gaudi2_validate_cb_address()
6684 asic_prop->pmmu_huge.end_addr)) in gaudi2_validate_cb_address()
6687 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { in gaudi2_validate_cb_address()
6688 if (!hdev->pdev) in gaudi2_validate_cb_address()
6691 if (!device_iommu_mapped(&hdev->pdev->dev)) in gaudi2_validate_cb_address()
6695 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", in gaudi2_validate_cb_address()
6696 parser->user_cb, parser->user_cb_size); in gaudi2_validate_cb_address()
6698 return -EFAULT; in gaudi2_validate_cb_address()
6703 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cs_parser()
6705 if (!parser->is_kernel_allocated_cb) in gaudi2_cs_parser()
6708 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_cs_parser()
6709 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); in gaudi2_cs_parser()
6710 return -EINVAL; in gaudi2_cs_parser()
6718 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_heartbeat()
6720 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_heartbeat()
6812 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; in gaudi2_send_job_to_kdma()
6813 cq_base = cq->kernel_address; in gaudi2_send_job_to_kdma()
6814 polling_addr = (u32 *)&cq_base[cq->ci]; in gaudi2_send_job_to_kdma()
6816 if (hdev->pldm) in gaudi2_send_job_to_kdma()
6835 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); in gaudi2_send_job_to_kdma()
6840 cq->ci = hl_cq_inc_ptr(cq->ci); in gaudi2_send_job_to_kdma()
6868 return hdev->asic_prop.first_available_user_sob[0] + in gaudi2_test_queue_hw_queue_id_to_sob_id()
6869 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; in gaudi2_test_queue_hw_queue_id_to_sob_id()
6886 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; in gaudi2_test_queue_send_msg_short()
6896 msg_short_pkt->value = cpu_to_le32(sob_val); in gaudi2_test_queue_send_msg_short()
6897 msg_short_pkt->ctl = cpu_to_le32(tmp); in gaudi2_test_queue_send_msg_short()
6899 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); in gaudi2_test_queue_send_msg_short()
6901 dev_err(hdev->dev, in gaudi2_test_queue_send_msg_short()
6914 if (hdev->pldm) in gaudi2_test_queue_wait_completion()
6927 if (rc == -ETIMEDOUT) { in gaudi2_test_queue_wait_completion()
6928 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", in gaudi2_test_queue_wait_completion()
6930 rc = -EIO; in gaudi2_test_queue_wait_completion()
6938 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_cpu_queue()
6944 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_test_cpu_queue()
6952 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues()
6962 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; in gaudi2_test_queues()
6981 /* chip is not usable, no need for cleanups, just bail-out with error */ in gaudi2_test_queues()
6994 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_compute_reset_late_init()
7002 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_compute_reset_late_init()
7009 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); in gaudi2_compute_reset_late_init()
7010 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); in gaudi2_compute_reset_late_init()
7017 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_edma_idle_status()
7019 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_edma_idle_status()
7027 "---- ---- ------- ------------ ------------- -------------\n"); in gaudi2_get_edma_idle_status()
7033 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_get_edma_idle_status()
7068 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_pdma_idle_status()
7076 "---- ------- ------------ ------------- -------------\n"); in gaudi2_get_pdma_idle_status()
7107 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; in gaudi2_get_nic_idle_status()
7114 if (e && hdev->nic_ports_mask) in gaudi2_get_nic_idle_status()
7117 "--- ------- ------------ ----------\n"); in gaudi2_get_nic_idle_status()
7125 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_get_nic_idle_status()
7154 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; in gaudi2_get_mme_idle_status()
7162 "--- ---- ------- ------------ ---------------\n"); in gaudi2_get_mme_idle_status()
7195 struct gaudi2_tpc_idle_data *idle_data = ctx->data; in gaudi2_is_tpc_engine_idle()
7200 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) in gaudi2_is_tpc_engine_idle()
7213 *(idle_data->is_idle) &= is_eng_idle; in gaudi2_is_tpc_engine_idle()
7215 if (idle_data->mask && !is_eng_idle) in gaudi2_is_tpc_engine_idle()
7216 set_bit(engine_idx, idle_data->mask); in gaudi2_is_tpc_engine_idle()
7218 if (idle_data->e) in gaudi2_is_tpc_engine_idle()
7219 hl_engine_data_sprintf(idle_data->e, in gaudi2_is_tpc_engine_idle()
7220 idle_data->tpc_fmt, dcore, inst, in gaudi2_is_tpc_engine_idle()
7228 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_tpc_idle_status()
7233 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", in gaudi2_get_tpc_idle_status()
7243 if (e && prop->tpc_enabled_mask) in gaudi2_get_tpc_idle_status()
7246 "---- --- ------- ------------ ---------- ------\n"); in gaudi2_get_tpc_idle_status()
7256 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_decoder_idle_status()
7258 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7259 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7266 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) in gaudi2_get_decoder_idle_status()
7269 "---- --- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7274 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) in gaudi2_get_decoder_idle_status()
7294 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) in gaudi2_get_decoder_idle_status()
7297 "-------- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7302 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) in gaudi2_get_decoder_idle_status()
7325 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; in gaudi2_get_rotator_idle_status()
7335 "---- --- ------- ------------ ------------ ----------\n"); in gaudi2_get_rotator_idle_status()
7377 __acquires(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_lock()
7379 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_lock()
7381 spin_lock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_lock()
7385 __releases(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_unlock()
7387 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_unlock()
7389 spin_unlock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_unlock()
7394 return hdev->pdev->device; in gaudi2_get_pci_id()
7399 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_eeprom_data()
7401 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_get_eeprom_data()
7414 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_events_stat()
7417 *size = (u32) sizeof(gaudi2->events_stat_aggregate); in gaudi2_get_events_stat()
7418 return gaudi2->events_stat_aggregate; in gaudi2_get_events_stat()
7421 *size = (u32) sizeof(gaudi2->events_stat); in gaudi2_get_events_stat()
7422 return gaudi2->events_stat; in gaudi2_get_events_stat()
7428 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * in gaudi2_mmu_vdec_dcore_prepare()
7451 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_dcore_prepare()
7458 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { in gaudi2_mmu_dcore_prepare()
7465 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { in gaudi2_mmu_dcore_prepare()
7475 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID in gaudi2_mmu_dcore_prepare()
7511 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) in gaudi2_mmu_dcore_prepare()
7519 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; in gudi2_mmu_vdec_shared_prepare()
7540 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; in gudi2_mmu_arc_farm_arc_dup_eng_prepare()
7552 /* Enable MMU and configure asid for all relevant ARC regions */ in gaudi2_arc_mmu_prepare()
7594 if (hdev->fw_components & FW_TYPE_BOOT_CPU) in gaudi2_arc_mmu_prepare_all()
7612 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_shared_prepare()
7640 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) in gaudi2_mmu_shared_prepare()
7643 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) in gaudi2_mmu_shared_prepare()
7660 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; in gaudi2_tpc_mmu_prepare()
7663 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7665 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7671 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_prepare()
7680 dev_crit(hdev->dev, "asid %u is too big\n", asid); in gaudi2_mmu_prepare()
7681 return -EINVAL; in gaudi2_mmu_prepare()
7684 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) in gaudi2_mmu_prepare()
7707 /* return in case of NIC status event - these events are received periodically and not as in is_info_event()
7728 dev_err_ratelimited(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7732 dev_err(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7745 ecc_address = le64_to_cpu(ecc_data->ecc_address); in gaudi2_handle_ecc_event()
7746 ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom); in gaudi2_handle_ecc_event()
7747 memory_wrapper_idx = ecc_data->memory_wrapper_idx; in gaudi2_handle_ecc_event()
7749 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, in gaudi2_handle_ecc_event()
7751 ecc_address, ecc_syndrom, memory_wrapper_idx, ecc_data->is_critical); in gaudi2_handle_ecc_event()
7753 return !!ecc_data->is_critical; in gaudi2_handle_ecc_event()
7775 dev_info(hdev->dev, in print_lower_qman_data_on_err()
7787 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
7788 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
7860 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_hbw_shared_printf_info()
7861 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", in gaudi2_razwi_rr_hbw_shared_printf_info()
7885 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_lbw_shared_printf_info()
7886 …"%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x… in gaudi2_razwi_rr_lbw_shared_printf_info()
7900 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
7903 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + in gaudi2_razwi_calc_engine_id()
7925 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
7953 if (hdev->tpc_binning) { in gaudi2_ack_module_razwi_event_handler()
7954 binned_idx = __ffs(hdev->tpc_binning); in gaudi2_ack_module_razwi_event_handler()
7962 !hdev->asic_prop.fw_security_enabled && in gaudi2_ack_module_razwi_event_handler()
8026 if (hdev->decoder_binning) { in gaudi2_ack_module_razwi_event_handler()
8027 binned_idx = __ffs(hdev->decoder_binning); in gaudi2_ack_module_razwi_event_handler()
8052 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); in gaudi2_ack_module_razwi_event_handler()
8097 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_check_if_razwi_happened()
8102 if (prop->tpc_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8114 if (prop->edma_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8123 if (hdev->nic_ports_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8129 if (prop->decoder_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8153 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", in gaudi2_psoc_razwi_get_engines()
8157 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", in gaudi2_psoc_razwi_get_engines()
8194 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8208 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8220 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8232 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8258 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { in gaudi2_ack_psoc_razwi_event_handler()
8266 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8277 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8282 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) in gaudi2_ack_psoc_razwi_event_handler()
8318 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8332 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_qm_sei_err()
8333 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_qm_sei_err()
8340 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8346 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_qm_sei_err()
8379 index = event_type - GAUDI2_EVENT_TPC0_QM; in gaudi2_handle_qman_err()
8384 index = event_type - GAUDI2_EVENT_TPC6_QM; in gaudi2_handle_qman_err()
8389 index = event_type - GAUDI2_EVENT_TPC12_QM; in gaudi2_handle_qman_err()
8394 index = event_type - GAUDI2_EVENT_TPC18_QM; in gaudi2_handle_qman_err()
8544 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_handle_rot_err()
8566 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_tpc_ack_interrupts()
8598 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); in gaudi2_handle_dec_err()
8852 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); in gaudi2_handle_page_error()
8857 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", in gaudi2_handle_page_error()
8884 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", in gaudi2_handle_access_error()
8965 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", in gaudi2_handle_sm_err()
9109 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9115 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); in gaudi2_hbm_sei_handle_read_err()
9116 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9126 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9128 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9130 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9131 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9133 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9135 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9137 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9138 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9143 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9145 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9148 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9149 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9155 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); in gaudi2_hbm_sei_handle_read_err()
9156 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9157 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); in gaudi2_hbm_sei_handle_read_err()
9158 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9159 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); in gaudi2_hbm_sei_handle_read_err()
9168 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; in gaudi2_hbm_sei_print_wr_par_info()
9169 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; in gaudi2_hbm_sei_print_wr_par_info()
9171 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_wr_par_info()
9173 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", in gaudi2_hbm_sei_print_wr_par_info()
9176 /* JIRA H6-3286 - the following prints may not be valid */ in gaudi2_hbm_sei_print_wr_par_info()
9177 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); in gaudi2_hbm_sei_print_wr_par_info()
9180 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_print_wr_par_info()
9193 __le32 *col_cmd = ca_par_err_data->dbg_col; in gaudi2_hbm_sei_print_ca_par_info()
9194 __le16 *row_cmd = ca_par_err_data->dbg_row; in gaudi2_hbm_sei_print_ca_par_info()
9197 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_ca_par_info()
9199 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); in gaudi2_hbm_sei_print_ca_par_info()
9201 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, in gaudi2_hbm_sei_print_ca_par_info()
9213 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; in gaudi2_handle_hbm_mc_sei_err()
9214 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; in gaudi2_handle_hbm_mc_sei_err()
9216 cause_idx = sei_data->hdr.sei_cause; in gaudi2_handle_hbm_mc_sei_err()
9217 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { in gaudi2_handle_hbm_mc_sei_err()
9224 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, in gaudi2_handle_hbm_mc_sei_err()
9225 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", in gaudi2_handle_hbm_mc_sei_err()
9226 sei_data->hdr.is_critical ? "Critical" : "Non-critical", in gaudi2_handle_hbm_mc_sei_err()
9227 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, in gaudi2_handle_hbm_mc_sei_err()
9230 /* Print error-specific info */ in gaudi2_handle_hbm_mc_sei_err()
9237 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, in gaudi2_handle_hbm_mc_sei_err()
9238 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9243 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, in gaudi2_handle_hbm_mc_sei_err()
9244 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9249 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, in gaudi2_handle_hbm_mc_sei_err()
9250 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9259 &sei_data->read_err_info, in gaudi2_handle_hbm_mc_sei_err()
9260 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9267 require_hard_reset |= !!sei_data->hdr.is_critical; in gaudi2_handle_hbm_mc_sei_err()
9290 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", in gaudi2_handle_hbm_mc_spi()
9302 mutex_lock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9306 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9307 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9308 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); in gaudi2_print_clk_change_info()
9309 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; in gaudi2_print_clk_change_info()
9310 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); in gaudi2_print_clk_change_info()
9314 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9315 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); in gaudi2_print_clk_change_info()
9316 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9320 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9321 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9322 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); in gaudi2_print_clk_change_info()
9323 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; in gaudi2_print_clk_change_info()
9325 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); in gaudi2_print_clk_change_info()
9329 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9330 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); in gaudi2_print_clk_change_info()
9332 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9336 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); in gaudi2_print_clk_change_info()
9340 mutex_unlock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9346 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_out_of_sync_info()
9350 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), in gaudi2_print_out_of_sync_info()
9351 q->pi, atomic_read(&q->ci)); in gaudi2_print_out_of_sync_info()
9372 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", in gaudi2_handle_pcie_p2p_msix()
9387 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); in gaudi2_handle_pcie_drain()
9388 lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw); in gaudi2_handle_pcie_drain()
9389 lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw); in gaudi2_handle_pcie_drain()
9390 hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw); in gaudi2_handle_pcie_drain()
9391 hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw); in gaudi2_handle_pcie_drain()
9394 dev_err_ratelimited(hdev->dev, in gaudi2_handle_pcie_drain()
9401 dev_err_ratelimited(hdev->dev, in gaudi2_handle_pcie_drain()
9417 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", in gaudi2_handle_psoc_drain()
9431 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_cpu_pkt_failure_info()
9435 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); in gaudi2_print_cpu_pkt_failure_info()
9445 intr_type = le32_to_cpu(data->intr_type); in hl_arc_event_handle()
9446 engine_id = le32_to_cpu(data->engine_id); in hl_arc_event_handle()
9447 payload = le64_to_cpu(data->payload); in hl_arc_event_handle()
9455 engine_id, intr_type, q->queue_index); in hl_arc_event_handle()
9465 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_handle_eqe()
9471 ctl = le32_to_cpu(eq_entry->hdr.ctl); in gaudi2_handle_eqe()
9475 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in gaudi2_handle_eqe()
9476 event_type, GAUDI2_EVENT_SIZE - 1); in gaudi2_handle_eqe()
9480 gaudi2->events_stat[event_type]++; in gaudi2_handle_eqe()
9481 gaudi2->events_stat_aggregate[event_type]++; in gaudi2_handle_eqe()
9489 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); in gaudi2_handle_eqe()
9490 is_critical = eq_entry->ecc_data.is_critical; in gaudi2_handle_eqe()
9522 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
9524 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9530 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_eqe()
9532 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9538 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; in gaudi2_handle_eqe()
9568 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / in gaudi2_handle_eqe()
9569 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); in gaudi2_handle_eqe()
9571 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9585 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / in gaudi2_handle_eqe()
9586 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); in gaudi2_handle_eqe()
9595 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_eqe()
9596 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_eqe()
9607 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / in gaudi2_handle_eqe()
9608 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - in gaudi2_handle_eqe()
9618 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / in gaudi2_handle_eqe()
9619 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - in gaudi2_handle_eqe()
9628 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9634 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9640 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9646 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); in gaudi2_handle_eqe()
9662 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9669 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9681 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { in gaudi2_handle_eqe()
9690 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9696 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9701 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); in gaudi2_handle_eqe()
9708 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9728 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9818 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
9827 /* Do nothing- FW will handle it */ in gaudi2_handle_eqe()
9836 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
9847 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
9848 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
9853 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
9854 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
9860 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
9867 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); in gaudi2_handle_eqe()
9880 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", in gaudi2_handle_eqe()
9902 if (hdev->hard_reset_on_fw_events || in gaudi2_handle_eqe()
9903 (hdev->asic_prop.fw_security_enabled && is_critical)) in gaudi2_handle_eqe()
9917 if (hdev->asic_prop.fw_security_enabled && is_critical) { in gaudi2_handle_eqe()
9943 lin_dma_pkt->ctl = cpu_to_le32(ctl); in gaudi2_memset_memory_chunk_using_edma_qm()
9944 lin_dma_pkt->src_addr = cpu_to_le64(val); in gaudi2_memset_memory_chunk_using_edma_qm()
9945 lin_dma_pkt->dst_addr = cpu_to_le64(addr); in gaudi2_memset_memory_chunk_using_edma_qm()
9946 lin_dma_pkt->tsize = cpu_to_le32(size); in gaudi2_memset_memory_chunk_using_edma_qm()
9952 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n", in gaudi2_memset_memory_chunk_using_edma_qm()
9967 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_memset_device_memory()
9972 if (prop->edma_enabled_mask == 0) { in gaudi2_memset_device_memory()
9973 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); in gaudi2_memset_device_memory()
9974 return -EIO; in gaudi2_memset_device_memory()
9977 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_memset_device_memory()
9992 return -ENOMEM; in gaudi2_memset_device_memory()
9995 * set mmu bypass for the scrubbing - all ddmas are configured the same so save in gaudi2_memset_device_memory()
10006 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10029 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10032 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); in gaudi2_memset_device_memory()
10052 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n"); in gaudi2_memset_device_memory()
10061 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10082 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_dram()
10083 u64 size = prop->dram_end_address - prop->dram_user_base_address; in gaudi2_scrub_device_dram()
10085 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); in gaudi2_scrub_device_dram()
10088 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", in gaudi2_scrub_device_dram()
10089 prop->dram_user_base_address, size); in gaudi2_scrub_device_dram()
10096 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_mem()
10097 u64 val = hdev->memory_scrub_val; in gaudi2_scrub_device_mem()
10100 if (!hdev->memory_scrub) in gaudi2_scrub_device_mem()
10104 addr = prop->sram_user_base_address; in gaudi2_scrub_device_mem()
10105 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); in gaudi2_scrub_device_mem()
10106 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", in gaudi2_scrub_device_mem()
10110 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10117 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10130 offset = hdev->asic_prop.first_available_cq[0] * 4; in gaudi2_restore_user_sm_registers()
10137 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - in gaudi2_restore_user_sm_registers()
10154 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; in gaudi2_restore_user_sm_registers()
10172 offset = hdev->asic_prop.first_available_user_mon[0] * 4; in gaudi2_restore_user_sm_registers()
10175 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); in gaudi2_restore_user_sm_registers()
10185 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; in gaudi2_restore_user_sm_registers()
10194 offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_restore_user_sm_registers()
10197 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - in gaudi2_restore_user_sm_registers()
10204 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; in gaudi2_restore_user_sm_registers()
10265 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; in gaudi2_init_block_instances()
10269 for (i = 0 ; i < cfg_ctx->instances ; i++) { in gaudi2_init_block_instances()
10270 seq = block_idx * cfg_ctx->instances + i; in gaudi2_init_block_instances()
10273 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) in gaudi2_init_block_instances()
10276 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, in gaudi2_init_block_instances()
10277 cfg_ctx->data); in gaudi2_init_block_instances()
10286 cfg_ctx->enabled_mask = mask; in gaudi2_init_blocks_with_mask()
10288 for (i = 0 ; i < cfg_ctx->blocks ; i++) in gaudi2_init_blocks_with_mask()
10309 dev_err(hdev->dev, "No ctx available\n"); in gaudi2_debugfs_read_dma()
10310 return -EINVAL; in gaudi2_debugfs_read_dma()
10317 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); in gaudi2_debugfs_read_dma()
10318 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10326 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); in gaudi2_debugfs_read_dma()
10327 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10332 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10336 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); in gaudi2_debugfs_read_dma()
10342 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10348 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10351 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); in gaudi2_debugfs_read_dma()
10372 size_left -= SZ_2M; in gaudi2_debugfs_read_dma()
10377 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10384 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10387 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10399 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_init()
10402 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_init()
10405 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, in gaudi2_internal_cb_pool_init()
10407 &hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10410 if (!hdev->internal_cb_pool_virt_addr) in gaudi2_internal_cb_pool_init()
10411 return -ENOMEM; in gaudi2_internal_cb_pool_init()
10416 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); in gaudi2_internal_cb_pool_init()
10417 if (!hdev->internal_cb_pool) { in gaudi2_internal_cb_pool_init()
10418 dev_err(hdev->dev, "Failed to create internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10419 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10423 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10424 HOST_SPACE_INTERNAL_CB_SZ, -1); in gaudi2_internal_cb_pool_init()
10426 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10427 rc = -EFAULT; in gaudi2_internal_cb_pool_init()
10431 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, in gaudi2_internal_cb_pool_init()
10434 if (!hdev->internal_cb_va_base) { in gaudi2_internal_cb_pool_init()
10435 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10439 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10441 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10450 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10455 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10457 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10458 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10460 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_init()
10462 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10463 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_init()
10470 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_fini()
10472 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_fini()
10475 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10476 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10477 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10479 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10481 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_fini()
10483 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_fini()
10484 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_fini()
10495 struct hl_device *hdev = ctx->hdev; in gaudi2_map_virtual_msix_doorbell_memory()
10496 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_map_virtual_msix_doorbell_memory()
10497 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_map_virtual_msix_doorbell_memory()
10501 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); in gaudi2_map_virtual_msix_doorbell_memory()
10503 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", in gaudi2_map_virtual_msix_doorbell_memory()
10511 struct hl_device *hdev = ctx->hdev; in gaudi2_unmap_virtual_msix_doorbell_memory()
10512 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_unmap_virtual_msix_doorbell_memory()
10516 prop->pmmu.page_size, true); in gaudi2_unmap_virtual_msix_doorbell_memory()
10518 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", in gaudi2_unmap_virtual_msix_doorbell_memory()
10526 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); in gaudi2_ctx_init()
10533 if (ctx->hdev->reset_upon_device_release) in gaudi2_ctx_init()
10534 gaudi2_restore_nic_qm_registers(ctx->hdev); in gaudi2_ctx_init()
10536 gaudi2_restore_user_registers(ctx->hdev); in gaudi2_ctx_init()
10538 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); in gaudi2_ctx_init()
10544 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_init()
10551 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi2_ctx_fini()
10554 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_fini()
10561 struct hl_device *hdev = cs->ctx->hdev; in gaudi2_pre_schedule_cs()
10562 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); in gaudi2_pre_schedule_cs()
10573 * generates MSI-X interrupt. in gaudi2_pre_schedule_cs()
10581 cs->jobs_cnt); in gaudi2_pre_schedule_cs()
10597 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); in gaudi2_gen_signal_cb()
10610 pkt->value = cpu_to_le32(value); in gaudi2_gen_signal_cb()
10611 pkt->ctl = cpu_to_le32(ctl); in gaudi2_gen_signal_cb()
10628 pkt->value = cpu_to_le32(value); in gaudi2_add_mon_msg_short()
10629 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_mon_msg_short()
10641 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); in gaudi2_add_arm_monitor_pkt()
10658 pkt->value = cpu_to_le32(value); in gaudi2_add_arm_monitor_pkt()
10659 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_arm_monitor_pkt()
10678 pkt->cfg = cpu_to_le32(cfg); in gaudi2_add_fence_pkt()
10679 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_fence_pkt()
10686 struct hl_cb *cb = prop->data; in gaudi2_gen_wait_cb()
10687 void *buf = (void *) (uintptr_t) (cb->kernel_address); in gaudi2_gen_wait_cb()
10690 u32 stream_index, size = prop->size; in gaudi2_gen_wait_cb()
10693 stream_index = prop->q_idx % 4; in gaudi2_gen_wait_cb()
10694 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + in gaudi2_gen_wait_cb()
10704 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
10710 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
10719 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
10725 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; in gaudi2_gen_wait_cb()
10727 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, in gaudi2_gen_wait_cb()
10728 prop->sob_val, msg_addr_offset); in gaudi2_gen_wait_cb()
10740 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); in gaudi2_reset_sob()
10742 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); in gaudi2_reset_sob()
10744 kref_init(&hw_sob->kref); in gaudi2_reset_sob()
10767 return -EINVAL; in gaudi2_collective_wait_create_jobs()
10771 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
10772 * to DMMU page-size address (64MB) before mapping it in
10784 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1
10785 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3
10790 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_scramble_addr()
10798 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_scramble_addr()
10810 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_descramble_addr()
10818 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_descramble_addr()
10834 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); in gaudi2_get_dec_base_addr()
10855 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_hw_block_id()
10859 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { in gaudi2_get_hw_block_id()
10862 *block_size = gaudi2->mapped_blocks[i].size; in gaudi2_get_hw_block_id()
10867 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); in gaudi2_get_hw_block_id()
10869 return -EINVAL; in gaudi2_get_hw_block_id()
10875 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_block_mmap()
10881 dev_err(hdev->dev, "Invalid block id %u", block_id); in gaudi2_block_mmap()
10882 return -EINVAL; in gaudi2_block_mmap()
10886 if (block_size != gaudi2->mapped_blocks[block_id].size) { in gaudi2_block_mmap()
10887 dev_err(hdev->dev, "Invalid block size %u", block_size); in gaudi2_block_mmap()
10888 return -EINVAL; in gaudi2_block_mmap()
10891 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; in gaudi2_block_mmap()
10893 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; in gaudi2_block_mmap()
10898 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, in gaudi2_block_mmap()
10899 block_size, vma->vm_page_prot); in gaudi2_block_mmap()
10901 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_block_mmap()
10908 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_events_from_fw()
10910 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_enable_events_from_fw()
10911 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); in gaudi2_enable_events_from_fw()
10913 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_enable_events_from_fw()
10973 return -EINVAL; in gaudi2_get_mmu_base()
10982 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_ack_mmu_error()
10985 if (!(gaudi2->hw_cap_initialized & mmu_id)) in gaudi2_ack_mmu_error()
11034 default: return -EINVAL; in gaudi2_map_pll_idx_to_fw_idx()
11077 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; in gaudi2_state_dump_init()
11078 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; in gaudi2_state_dump_init()
11101 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_get_real_page_size()
11105 if (page_size % mmu_prop->page_size) in gaudi2_mmu_get_real_page_size()
11108 *real_page_size = mmu_prop->page_size; in gaudi2_mmu_get_real_page_size()
11112 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) in gaudi2_mmu_get_real_page_size()
11123 *real_page_size = prop->dram_page_size; in gaudi2_mmu_get_real_page_size()
11128 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", in gaudi2_mmu_get_real_page_size()
11129 page_size, mmu_prop->page_size >> 10); in gaudi2_mmu_get_real_page_size()
11130 return -EFAULT; in gaudi2_mmu_get_real_page_size()
11135 return -EOPNOTSUPP; in gaudi2_get_monitor_dump()
11140 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_device_activity()
11142 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_device_activity()
11252 hdev->asic_funcs = &gaudi2_funcs; in gaudi2_set_asic_funcs()