Lines Matching +full:max +full:- +full:outbound +full:- +full:regions
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
23 * - Range registers (When MMU is enabled, DMA RR does NOT protect host)
24 * - MMU
27 * - Range registers (protect the first 512MB)
28 * - MMU (isolation between users)
31 * - Range registers
32 * - Protection bits
44 * - checks DMA pointer
45 * - WREG, MSG_PROT are not allowed.
46 * - MSG_LONG/SHORT are allowed.
57 * - PQ entry is in kernel address space and the driver doesn't map it.
58 * - CP writes to MSIX register and to kernel address space (completion
73 #define GOYA_BOOT_FIT_FILE "habanalabs/goya/goya-boot-fit.itb"
74 #define GOYA_LINUX_FW_FILE "habanalabs/goya/goya-fit.itb"
362 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_set_fixed_properties()
365 prop->max_queues = GOYA_QUEUE_ID_SIZE; in goya_set_fixed_properties()
366 prop->hw_queues_props = kcalloc(prop->max_queues, in goya_set_fixed_properties()
370 if (!prop->hw_queues_props) in goya_set_fixed_properties()
371 return -ENOMEM; in goya_set_fixed_properties()
374 prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; in goya_set_fixed_properties()
375 prop->hw_queues_props[i].driver_only = 0; in goya_set_fixed_properties()
376 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; in goya_set_fixed_properties()
380 prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; in goya_set_fixed_properties()
381 prop->hw_queues_props[i].driver_only = 1; in goya_set_fixed_properties()
382 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; in goya_set_fixed_properties()
387 prop->hw_queues_props[i].type = QUEUE_TYPE_INT; in goya_set_fixed_properties()
388 prop->hw_queues_props[i].driver_only = 0; in goya_set_fixed_properties()
389 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER; in goya_set_fixed_properties()
392 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE; in goya_set_fixed_properties()
393 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; in goya_set_fixed_properties()
395 prop->dram_base_address = DRAM_PHYS_BASE; in goya_set_fixed_properties()
396 prop->dram_size = DRAM_PHYS_DEFAULT_SIZE; in goya_set_fixed_properties()
397 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in goya_set_fixed_properties()
398 prop->dram_user_base_address = DRAM_BASE_ADDR_USER; in goya_set_fixed_properties()
400 prop->sram_base_address = SRAM_BASE_ADDR; in goya_set_fixed_properties()
401 prop->sram_size = SRAM_SIZE; in goya_set_fixed_properties()
402 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in goya_set_fixed_properties()
403 prop->sram_user_base_address = prop->sram_base_address + in goya_set_fixed_properties()
406 prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR; in goya_set_fixed_properties()
407 prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR; in goya_set_fixed_properties()
408 if (hdev->pldm) in goya_set_fixed_properties()
409 prop->mmu_pgt_size = 0x800000; /* 8MB */ in goya_set_fixed_properties()
411 prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE; in goya_set_fixed_properties()
412 prop->mmu_pte_size = HL_PTE_SIZE; in goya_set_fixed_properties()
413 prop->mmu_hop_table_size = HOP_TABLE_SIZE; in goya_set_fixed_properties()
414 prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE; in goya_set_fixed_properties()
415 prop->dram_page_size = PAGE_SIZE_2MB; in goya_set_fixed_properties()
416 prop->dram_supports_virtual_memory = true; in goya_set_fixed_properties()
418 prop->dmmu.hop0_shift = HOP0_SHIFT; in goya_set_fixed_properties()
419 prop->dmmu.hop1_shift = HOP1_SHIFT; in goya_set_fixed_properties()
420 prop->dmmu.hop2_shift = HOP2_SHIFT; in goya_set_fixed_properties()
421 prop->dmmu.hop3_shift = HOP3_SHIFT; in goya_set_fixed_properties()
422 prop->dmmu.hop4_shift = HOP4_SHIFT; in goya_set_fixed_properties()
423 prop->dmmu.hop0_mask = HOP0_MASK; in goya_set_fixed_properties()
424 prop->dmmu.hop1_mask = HOP1_MASK; in goya_set_fixed_properties()
425 prop->dmmu.hop2_mask = HOP2_MASK; in goya_set_fixed_properties()
426 prop->dmmu.hop3_mask = HOP3_MASK; in goya_set_fixed_properties()
427 prop->dmmu.hop4_mask = HOP4_MASK; in goya_set_fixed_properties()
428 prop->dmmu.start_addr = VA_DDR_SPACE_START; in goya_set_fixed_properties()
429 prop->dmmu.end_addr = VA_DDR_SPACE_END; in goya_set_fixed_properties()
430 prop->dmmu.page_size = PAGE_SIZE_2MB; in goya_set_fixed_properties()
431 prop->dmmu.num_hops = MMU_ARCH_5_HOPS; in goya_set_fixed_properties()
434 memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu)); in goya_set_fixed_properties()
435 prop->pmmu.start_addr = VA_HOST_SPACE_START; in goya_set_fixed_properties()
436 prop->pmmu.end_addr = VA_HOST_SPACE_END; in goya_set_fixed_properties()
437 prop->pmmu.page_size = PAGE_SIZE_4KB; in goya_set_fixed_properties()
438 prop->pmmu.num_hops = MMU_ARCH_5_HOPS; in goya_set_fixed_properties()
441 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in goya_set_fixed_properties()
442 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in goya_set_fixed_properties()
444 prop->dram_size_for_default_page_mapping = VA_DDR_SPACE_END; in goya_set_fixed_properties()
445 prop->cfg_size = CFG_SIZE; in goya_set_fixed_properties()
446 prop->max_asid = MAX_ASID; in goya_set_fixed_properties()
447 prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE; in goya_set_fixed_properties()
448 prop->high_pll = PLL_HIGH_DEFAULT; in goya_set_fixed_properties()
449 prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT; in goya_set_fixed_properties()
450 prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE; in goya_set_fixed_properties()
451 prop->max_power_default = MAX_POWER_DEFAULT; in goya_set_fixed_properties()
452 prop->dc_power_default = DC_POWER_DEFAULT; in goya_set_fixed_properties()
453 prop->tpc_enabled_mask = TPC_ENABLED_MASK; in goya_set_fixed_properties()
454 prop->pcie_dbi_base_address = mmPCIE_DBI_BASE; in goya_set_fixed_properties()
455 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in goya_set_fixed_properties()
457 strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, in goya_set_fixed_properties()
460 prop->max_pending_cs = GOYA_MAX_PENDING_CS; in goya_set_fixed_properties()
462 prop->first_available_user_msix_interrupt = USHRT_MAX; in goya_set_fixed_properties()
465 prop->first_available_cq[i] = USHRT_MAX; in goya_set_fixed_properties()
467 prop->fw_cpu_boot_dev_sts0_valid = false; in goya_set_fixed_properties()
468 prop->fw_cpu_boot_dev_sts1_valid = false; in goya_set_fixed_properties()
469 prop->hard_reset_done_by_fw = false; in goya_set_fixed_properties()
470 prop->gic_interrupts_enable = true; in goya_set_fixed_properties()
472 prop->server_type = HL_SERVER_TYPE_UNKNOWN; in goya_set_fixed_properties()
478 * goya_pci_bars_map - Map PCI BARS of Goya device
482 * Request PCI regions and map them to kernel virtual addresses.
496 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_pci_bars_map()
497 (CFG_BASE - SRAM_BASE_ADDR); in goya_pci_bars_map()
504 struct goya_device *goya = hdev->asic_specific; in goya_set_ddr_bar_base()
509 if ((goya) && (goya->ddr_bar_cur_addr == addr)) in goya_set_ddr_bar_base()
512 /* Inbound Region 1 - Bar 4 - Point to DDR */ in goya_set_ddr_bar_base()
521 old_addr = goya->ddr_bar_cur_addr; in goya_set_ddr_bar_base()
522 goya->ddr_bar_cur_addr = addr; in goya_set_ddr_bar_base()
529 * goya_init_iatu - Initialize the iATU unit inside the PCI controller
542 if (hdev->asic_prop.iatu_done_by_fw) in goya_init_iatu()
545 /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */ in goya_init_iatu()
553 /* Inbound Region 1 - Bar 4 - Point to DDR */ in goya_init_iatu()
561 hdev->asic_funcs->set_dma_mask_from_fw(hdev); in goya_init_iatu()
563 /* Outbound Region 0 - Point to Host */ in goya_init_iatu()
578 * goya_early_init - GOYA early initialization code
590 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_early_init()
591 struct pci_dev *pdev = hdev->pdev; in goya_early_init()
597 dev_err(hdev->dev, "Failed to get fixed properties\n"); in goya_early_init()
603 dev_err(hdev->dev, in goya_early_init()
609 rc = -ENODEV; in goya_early_init()
614 dev_err(hdev->dev, in goya_early_init()
620 rc = -ENODEV; in goya_early_init()
624 prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID); in goya_early_init()
627 if (hdev->asic_prop.fw_security_enabled) { in goya_early_init()
628 hdev->asic_prop.iatu_done_by_fw = true; in goya_early_init()
640 hdev->asic_prop.iatu_done_by_fw = true; in goya_early_init()
648 * version to determine whether we run with a security-enabled firmware in goya_early_init()
656 if (hdev->reset_on_preboot_fail) in goya_early_init()
657 hdev->asic_funcs->hw_fini(hdev, true, false); in goya_early_init()
662 dev_info(hdev->dev, in goya_early_init()
664 hdev->asic_funcs->hw_fini(hdev, true, false); in goya_early_init()
667 if (!hdev->pldm) { in goya_early_init()
670 dev_warn(hdev->dev, in goya_early_init()
679 kfree(hdev->asic_prop.hw_queues_props); in goya_early_init()
684 * goya_early_fini - GOYA early finalization code
693 kfree(hdev->asic_prop.hw_queues_props); in goya_early_fini()
708 struct goya_device *goya = hdev->asic_specific; in goya_qman0_set_security()
710 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_qman0_set_security()
722 * goya_fetch_psoc_frequency - Fetch PSOC frequency values
729 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_fetch_psoc_frequency()
734 if (hdev->asic_prop.fw_security_enabled) { in goya_fetch_psoc_frequency()
764 dev_warn(hdev->dev, in goya_fetch_psoc_frequency()
771 prop->psoc_timestamp_frequency = freq; in goya_fetch_psoc_frequency()
772 prop->psoc_pci_pll_nr = nr; in goya_fetch_psoc_frequency()
773 prop->psoc_pci_pll_nf = nf; in goya_fetch_psoc_frequency()
774 prop->psoc_pci_pll_od = od; in goya_fetch_psoc_frequency()
775 prop->psoc_pci_pll_div_factor = div_fctr; in goya_fetch_psoc_frequency()
780 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_late_init()
787 dev_err(hdev->dev, in goya_late_init()
794 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); in goya_late_init()
812 dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc); in goya_late_init()
820 WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size)); in goya_late_init()
824 dev_err(hdev->dev, in goya_late_init()
833 * goya_late_fini - GOYA late tear-down code
844 if (!hdev->hl_chip_info->info) in goya_late_fini()
847 channel_info_arr = hdev->hl_chip_info->info; in goya_late_fini()
850 kfree(channel_info_arr[i]->config); in goya_late_fini()
857 hdev->hl_chip_info->info = NULL; in goya_late_fini()
862 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_set_pci_memory_regions()
866 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in goya_set_pci_memory_regions()
867 region->region_base = CFG_BASE; in goya_set_pci_memory_regions()
868 region->region_size = CFG_SIZE; in goya_set_pci_memory_regions()
869 region->offset_in_bar = CFG_BASE - SRAM_BASE_ADDR; in goya_set_pci_memory_regions()
870 region->bar_size = CFG_BAR_SIZE; in goya_set_pci_memory_regions()
871 region->bar_id = SRAM_CFG_BAR_ID; in goya_set_pci_memory_regions()
872 region->used = 1; in goya_set_pci_memory_regions()
875 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in goya_set_pci_memory_regions()
876 region->region_base = SRAM_BASE_ADDR; in goya_set_pci_memory_regions()
877 region->region_size = SRAM_SIZE; in goya_set_pci_memory_regions()
878 region->offset_in_bar = 0; in goya_set_pci_memory_regions()
879 region->bar_size = CFG_BAR_SIZE; in goya_set_pci_memory_regions()
880 region->bar_id = SRAM_CFG_BAR_ID; in goya_set_pci_memory_regions()
881 region->used = 1; in goya_set_pci_memory_regions()
884 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in goya_set_pci_memory_regions()
885 region->region_base = DRAM_PHYS_BASE; in goya_set_pci_memory_regions()
886 region->region_size = hdev->asic_prop.dram_size; in goya_set_pci_memory_regions()
887 region->offset_in_bar = 0; in goya_set_pci_memory_regions()
888 region->bar_size = prop->dram_pci_bar_size; in goya_set_pci_memory_regions()
889 region->bar_id = DDR_BAR_ID; in goya_set_pci_memory_regions()
890 region->used = 1; in goya_set_pci_memory_regions()
894 * goya_sw_init - Goya software initialization code
907 return -ENOMEM; in goya_sw_init()
910 goya->ddr_bar_cur_addr = DRAM_PHYS_BASE; in goya_sw_init()
912 goya->mme_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
913 goya->tpc_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
914 goya->ic_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
916 hdev->asic_specific = goya; in goya_sw_init()
919 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), in goya_sw_init()
920 &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0); in goya_sw_init()
921 if (!hdev->dma_pool) { in goya_sw_init()
922 dev_err(hdev->dev, "failed to create DMA pool\n"); in goya_sw_init()
923 rc = -ENOMEM; in goya_sw_init()
927 hdev->cpu_accessible_dma_mem = in goya_sw_init()
928 hdev->asic_funcs->asic_dma_alloc_coherent(hdev, in goya_sw_init()
930 &hdev->cpu_accessible_dma_address, in goya_sw_init()
933 if (!hdev->cpu_accessible_dma_mem) { in goya_sw_init()
934 rc = -ENOMEM; in goya_sw_init()
938 dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n", in goya_sw_init()
939 &hdev->cpu_accessible_dma_address); in goya_sw_init()
941 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in goya_sw_init()
942 if (!hdev->cpu_accessible_dma_pool) { in goya_sw_init()
943 dev_err(hdev->dev, in goya_sw_init()
945 rc = -ENOMEM; in goya_sw_init()
949 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, in goya_sw_init()
950 (uintptr_t) hdev->cpu_accessible_dma_mem, in goya_sw_init()
951 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in goya_sw_init()
953 dev_err(hdev->dev, in goya_sw_init()
955 rc = -EFAULT; in goya_sw_init()
959 spin_lock_init(&goya->hw_queues_lock); in goya_sw_init()
960 hdev->supports_coresight = true; in goya_sw_init()
961 hdev->supports_soft_reset = true; in goya_sw_init()
962 hdev->allow_external_soft_reset = true; in goya_sw_init()
963 hdev->supports_wait_for_multi_cs = false; in goya_sw_init()
965 hdev->asic_funcs->set_pci_memory_regions(hdev); in goya_sw_init()
970 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_init()
972 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_init()
974 hdev->cpu_accessible_dma_mem, in goya_sw_init()
975 hdev->cpu_accessible_dma_address); in goya_sw_init()
977 dma_pool_destroy(hdev->dma_pool); in goya_sw_init()
985 * goya_sw_fini - Goya software tear-down code
992 struct goya_device *goya = hdev->asic_specific; in goya_sw_fini()
994 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_fini()
996 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_fini()
998 hdev->cpu_accessible_dma_mem, in goya_sw_fini()
999 hdev->cpu_accessible_dma_address); in goya_sw_fini()
1001 dma_pool_destroy(hdev->dma_pool); in goya_sw_fini()
1011 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qman()
1015 u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI); in goya_init_dma_qman()
1048 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_init_dma_qman()
1053 if (hdev->stop_on_err) in goya_init_dma_qman()
1064 u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1); in goya_init_dma_ch()
1078 (dma_id - 1) * 4; in goya_init_dma_ch()
1087 * goya_init_dma_qmans - Initialize QMAN DMA registers
1096 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qmans()
1100 if (goya->hw_cap_initialized & HW_CAP_DMA) in goya_init_dma_qmans()
1103 q = &hdev->kernel_queues[0]; in goya_init_dma_qmans()
1106 q->cq_id = q->msi_vec = i; in goya_init_dma_qmans()
1107 goya_init_dma_qman(hdev, i, q->bus_address); in goya_init_dma_qmans()
1111 goya->hw_cap_initialized |= HW_CAP_DMA; in goya_init_dma_qmans()
1115 * goya_disable_external_queues - Disable external queues
1122 struct goya_device *goya = hdev->asic_specific; in goya_disable_external_queues()
1124 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_disable_external_queues()
1168 dev_err(hdev->dev, in goya_stop_queue()
1170 return -EINVAL; in goya_stop_queue()
1177 * goya_stop_external_queues - Stop external queues
1188 struct goya_device *goya = hdev->asic_specific; in goya_stop_external_queues()
1190 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_stop_external_queues()
1199 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n"); in goya_stop_external_queues()
1200 retval = -EIO; in goya_stop_external_queues()
1209 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n"); in goya_stop_external_queues()
1210 retval = -EIO; in goya_stop_external_queues()
1219 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n"); in goya_stop_external_queues()
1220 retval = -EIO; in goya_stop_external_queues()
1229 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n"); in goya_stop_external_queues()
1230 retval = -EIO; in goya_stop_external_queues()
1239 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n"); in goya_stop_external_queues()
1240 retval = -EIO; in goya_stop_external_queues()
1247 * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
1256 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu_queues()
1257 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_init_cpu_queues()
1260 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_init_cpu_queues()
1263 if (!hdev->cpu_queues_enable) in goya_init_cpu_queues()
1266 if (goya->hw_cap_initialized & HW_CAP_CPU_Q) in goya_init_cpu_queues()
1269 eq = &hdev->event_queue; in goya_init_cpu_queues()
1271 WREG32(mmCPU_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in goya_init_cpu_queues()
1272 WREG32(mmCPU_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in goya_init_cpu_queues()
1274 WREG32(mmCPU_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in goya_init_cpu_queues()
1275 WREG32(mmCPU_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in goya_init_cpu_queues()
1305 dev_err(hdev->dev, in goya_init_cpu_queues()
1307 return -EIO; in goya_init_cpu_queues()
1311 if (prop->fw_cpu_boot_dev_sts0_valid) in goya_init_cpu_queues()
1312 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); in goya_init_cpu_queues()
1314 if (prop->fw_cpu_boot_dev_sts1_valid) in goya_init_cpu_queues()
1315 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); in goya_init_cpu_queues()
1317 goya->hw_cap_initialized |= HW_CAP_CPU_Q; in goya_init_cpu_queues()
1373 tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE); in _goya_tpc_mbist_workaround()
1383 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n", in _goya_tpc_mbist_workaround()
1411 dev_err(hdev->dev, in _goya_tpc_mbist_workaround()
1432 struct goya_device *goya = hdev->asic_specific; in goya_tpc_mbist_workaround()
1435 if (hdev->pldm) in goya_tpc_mbist_workaround()
1438 if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST) in goya_tpc_mbist_workaround()
1446 goya->hw_cap_initialized |= HW_CAP_TPC_MBIST; in goya_tpc_mbist_workaround()
1450 * goya_init_golden_registers - Initialize golden registers
1459 struct goya_device *goya = hdev->asic_specific; in goya_init_golden_registers()
1463 if (goya->hw_cap_initialized & HW_CAP_GOLDEN) in goya_init_golden_registers()
1739 * Workaround for H2 #HW-23 bug in goya_init_golden_registers()
1740 * Set DMA max outstanding read requests to 240 on DMA CH 1. in goya_init_golden_registers()
1749 goya->hw_cap_initialized |= HW_CAP_GOLDEN; in goya_init_golden_registers()
1769 qman_base_addr = hdev->asic_prop.sram_base_address + in goya_init_mme_qman()
1840 struct goya_device *goya = hdev->asic_specific; in goya_init_mme_qmans()
1843 if (goya->hw_cap_initialized & HW_CAP_MME) in goya_init_mme_qmans()
1855 goya->hw_cap_initialized |= HW_CAP_MME; in goya_init_mme_qmans()
1864 u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI); in goya_init_tpc_qman()
1876 qman_base_addr = hdev->asic_prop.sram_base_address + base_off; in goya_init_tpc_qman()
1913 u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1); in goya_init_tpc_cmdq()
1947 struct goya_device *goya = hdev->asic_specific; in goya_init_tpc_qmans()
1949 u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW - in goya_init_tpc_qmans()
1953 if (goya->hw_cap_initialized & HW_CAP_TPC) in goya_init_tpc_qmans()
1978 goya->hw_cap_initialized |= HW_CAP_TPC; in goya_init_tpc_qmans()
1982 * goya_disable_internal_queues - Disable internal queues
1989 struct goya_device *goya = hdev->asic_specific; in goya_disable_internal_queues()
1991 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_disable_internal_queues()
1998 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_disable_internal_queues()
2027 * goya_stop_internal_queues - Stop internal queues
2036 struct goya_device *goya = hdev->asic_specific; in goya_stop_internal_queues()
2039 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_stop_internal_queues()
2054 dev_err(hdev->dev, "failed to stop MME QMAN\n"); in goya_stop_internal_queues()
2055 retval = -EIO; in goya_stop_internal_queues()
2064 dev_err(hdev->dev, "failed to stop MME CMDQ\n"); in goya_stop_internal_queues()
2065 retval = -EIO; in goya_stop_internal_queues()
2069 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_stop_internal_queues()
2078 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n"); in goya_stop_internal_queues()
2079 retval = -EIO; in goya_stop_internal_queues()
2088 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n"); in goya_stop_internal_queues()
2089 retval = -EIO; in goya_stop_internal_queues()
2098 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n"); in goya_stop_internal_queues()
2099 retval = -EIO; in goya_stop_internal_queues()
2108 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n"); in goya_stop_internal_queues()
2109 retval = -EIO; in goya_stop_internal_queues()
2118 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n"); in goya_stop_internal_queues()
2119 retval = -EIO; in goya_stop_internal_queues()
2128 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n"); in goya_stop_internal_queues()
2129 retval = -EIO; in goya_stop_internal_queues()
2138 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n"); in goya_stop_internal_queues()
2139 retval = -EIO; in goya_stop_internal_queues()
2148 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n"); in goya_stop_internal_queues()
2149 retval = -EIO; in goya_stop_internal_queues()
2158 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n"); in goya_stop_internal_queues()
2159 retval = -EIO; in goya_stop_internal_queues()
2168 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n"); in goya_stop_internal_queues()
2169 retval = -EIO; in goya_stop_internal_queues()
2178 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n"); in goya_stop_internal_queues()
2179 retval = -EIO; in goya_stop_internal_queues()
2188 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n"); in goya_stop_internal_queues()
2189 retval = -EIO; in goya_stop_internal_queues()
2198 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n"); in goya_stop_internal_queues()
2199 retval = -EIO; in goya_stop_internal_queues()
2208 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n"); in goya_stop_internal_queues()
2209 retval = -EIO; in goya_stop_internal_queues()
2218 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n"); in goya_stop_internal_queues()
2219 retval = -EIO; in goya_stop_internal_queues()
2228 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n"); in goya_stop_internal_queues()
2229 retval = -EIO; in goya_stop_internal_queues()
2237 struct goya_device *goya = hdev->asic_specific; in goya_dma_stall()
2239 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_dma_stall()
2251 struct goya_device *goya = hdev->asic_specific; in goya_tpc_stall()
2253 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_tpc_stall()
2268 struct goya_device *goya = hdev->asic_specific; in goya_mme_stall()
2270 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_mme_stall()
2278 struct goya_device *goya = hdev->asic_specific; in goya_enable_msix()
2279 int cq_cnt = hdev->asic_prop.completion_queues_count; in goya_enable_msix()
2282 if (goya->hw_cap_initialized & HW_CAP_MSIX) in goya_enable_msix()
2285 rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES, in goya_enable_msix()
2288 dev_err(hdev->dev, in goya_enable_msix()
2289 "MSI-X: Failed to enable support -- %d/%d\n", in goya_enable_msix()
2295 irq = pci_irq_vector(hdev->pdev, i); in goya_enable_msix()
2297 &hdev->completion_queue[i]); in goya_enable_msix()
2299 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2304 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_enable_msix()
2308 &hdev->event_queue); in goya_enable_msix()
2310 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2314 goya->hw_cap_initialized |= HW_CAP_MSIX; in goya_enable_msix()
2319 free_irq(pci_irq_vector(hdev->pdev, i), in goya_enable_msix()
2320 &hdev->completion_queue[i]); in goya_enable_msix()
2322 pci_free_irq_vectors(hdev->pdev); in goya_enable_msix()
2328 struct goya_device *goya = hdev->asic_specific; in goya_sync_irqs()
2331 if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) in goya_sync_irqs()
2335 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in goya_sync_irqs()
2336 synchronize_irq(pci_irq_vector(hdev->pdev, i)); in goya_sync_irqs()
2338 synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX)); in goya_sync_irqs()
2343 struct goya_device *goya = hdev->asic_specific; in goya_disable_msix()
2346 if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) in goya_disable_msix()
2351 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_disable_msix()
2352 free_irq(irq, &hdev->event_queue); in goya_disable_msix()
2354 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in goya_disable_msix()
2355 irq = pci_irq_vector(hdev->pdev, i); in goya_disable_msix()
2356 free_irq(irq, &hdev->completion_queue[i]); in goya_disable_msix()
2359 pci_free_irq_vectors(hdev->pdev); in goya_disable_msix()
2361 goya->hw_cap_initialized &= ~HW_CAP_MSIX; in goya_disable_msix()
2367 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in goya_enable_timestamp()
2369 /* Zero the lower/upper parts of the 64-bit counter */ in goya_enable_timestamp()
2370 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0); in goya_enable_timestamp()
2371 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0); in goya_enable_timestamp()
2374 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1); in goya_enable_timestamp()
2380 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in goya_disable_timestamp()
2387 dev_info(hdev->dev, in goya_halt_engines()
2390 if (hdev->pldm) in goya_halt_engines()
2420 * goya_load_firmware_to_device() - Load LINUX FW code to device.
2425 * Return: 0 on success, non-zero for failure.
2431 dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET; in goya_load_firmware_to_device()
2437 * goya_load_boot_fit_to_device() - Load boot fit to device.
2442 * Return: 0 on success, non-zero for failure.
2448 dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET; in goya_load_boot_fit_to_device()
2458 dynamic_loader = &hdev->fw_loader.dynamic_loader; in goya_init_dynamic_firmware_loader()
2463 * hard-coded) in later stages of the protocol those values will be in goya_init_dynamic_firmware_loader()
2465 * will always be up-to-date in goya_init_dynamic_firmware_loader()
2467 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; in goya_init_dynamic_firmware_loader()
2468 dyn_regs->kmd_msg_to_cpu = in goya_init_dynamic_firmware_loader()
2470 dyn_regs->cpu_cmd_status_to_host = in goya_init_dynamic_firmware_loader()
2473 dynamic_loader->wait_for_bl_timeout = GOYA_WAIT_FOR_BL_TIMEOUT_USEC; in goya_init_dynamic_firmware_loader()
2480 static_loader = &hdev->fw_loader.static_loader; in goya_init_static_firmware_loader()
2482 static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; in goya_init_static_firmware_loader()
2483 static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; in goya_init_static_firmware_loader()
2484 static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; in goya_init_static_firmware_loader()
2485 static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; in goya_init_static_firmware_loader()
2486 static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in goya_init_static_firmware_loader()
2487 static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; in goya_init_static_firmware_loader()
2488 static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; in goya_init_static_firmware_loader()
2489 static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; in goya_init_static_firmware_loader()
2490 static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; in goya_init_static_firmware_loader()
2491 static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; in goya_init_static_firmware_loader()
2492 static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; in goya_init_static_firmware_loader()
2493 static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); in goya_init_static_firmware_loader()
2498 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_init_firmware_loader()
2499 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in goya_init_firmware_loader()
2502 fw_loader->linux_loaded = false; in goya_init_firmware_loader()
2503 fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE; in goya_init_firmware_loader()
2504 fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE; in goya_init_firmware_loader()
2505 fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC; in goya_init_firmware_loader()
2506 fw_loader->boot_fit_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC; in goya_init_firmware_loader()
2507 fw_loader->skip_bmc = false; in goya_init_firmware_loader()
2508 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; in goya_init_firmware_loader()
2509 fw_loader->dram_bar_id = DDR_BAR_ID; in goya_init_firmware_loader()
2511 if (prop->dynamic_fw_load) in goya_init_firmware_loader()
2519 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu()
2522 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in goya_init_cpu()
2525 if (goya->hw_cap_initialized & HW_CAP_CPU) in goya_init_cpu()
2529 * Before pushing u-boot/linux to device, need to set the ddr bar to in goya_init_cpu()
2533 dev_err(hdev->dev, in goya_init_cpu()
2535 return -EIO; in goya_init_cpu()
2543 goya->hw_cap_initialized |= HW_CAP_CPU; in goya_init_cpu()
2554 if (hdev->pldm) in goya_mmu_update_asid_hop0_addr()
2572 dev_err(hdev->dev, in goya_mmu_update_asid_hop0_addr()
2582 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_init()
2583 struct goya_device *goya = hdev->asic_specific; in goya_mmu_init()
2587 if (!hdev->mmu_enable) in goya_mmu_init()
2590 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_mmu_init()
2593 hdev->dram_default_page_mapping = true; in goya_mmu_init()
2595 for (i = 0 ; i < prop->max_asid ; i++) { in goya_mmu_init()
2596 hop0_addr = prop->mmu_pgt_addr + in goya_mmu_init()
2597 (i * prop->mmu_hop_table_size); in goya_mmu_init()
2601 dev_err(hdev->dev, in goya_mmu_init()
2607 goya->hw_cap_initialized |= HW_CAP_MMU; in goya_mmu_init()
2618 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, in goya_mmu_init()
2631 * goya_hw_init - Goya hardware initialization code
2640 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_hw_init()
2656 dev_err(hdev->dev, "failed to initialize CPU\n"); in goya_hw_init()
2669 ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) { in goya_hw_init()
2670 dev_err(hdev->dev, in goya_hw_init()
2672 return -EIO; in goya_hw_init()
2689 /* MSI-X must be enabled before CPU queues are initialized */ in goya_hw_init()
2694 /* Perform read from the device to flush all MSI-X configuration */ in goya_hw_init()
2708 struct goya_device *goya = hdev->asic_specific; in goya_hw_fini()
2711 if (hdev->pldm) { in goya_hw_fini()
2734 dev_info(hdev->dev, in goya_hw_fini()
2739 dev_info(hdev->dev, in goya_hw_fini()
2753 dev_err(hdev->dev, in goya_hw_fini()
2758 goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME | in goya_hw_fini()
2765 /* Chicken bit to re-initiate boot sequencer flow */ in goya_hw_fini()
2773 goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | in goya_hw_fini()
2779 memset(goya->events_stat, 0, sizeof(goya->events_stat)); in goya_hw_fini()
2789 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); in goya_suspend()
2804 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | in goya_mmap()
2807 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, in goya_mmap()
2808 (dma_addr - HOST_PHYS_BASE), size); in goya_mmap()
2810 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in goya_mmap()
2882 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", in goya_ring_doorbell()
2909 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, in goya_dma_alloc_coherent()
2923 dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE; in goya_dma_free_coherent()
2925 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); in goya_dma_free_coherent()
2939 *dma_handle = hdev->asic_prop.sram_base_address; in goya_get_int_queue_base()
2941 base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; in goya_get_int_queue_base()
2981 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); in goya_get_int_queue_base()
3000 if (hdev->pldm) in goya_send_job_on_qman0()
3005 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { in goya_send_job_on_qman0()
3006 dev_err_ratelimited(hdev->dev, in goya_send_job_on_qman0()
3008 return -EBUSY; in goya_send_job_on_qman0()
3011 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_send_job_on_qman0()
3014 dev_err(hdev->dev, in goya_send_job_on_qman0()
3016 return -ENOMEM; in goya_send_job_on_qman0()
3021 cb = job->patched_cb; in goya_send_job_on_qman0()
3023 fence_pkt = cb->kernel_address + in goya_send_job_on_qman0()
3024 job->job_cb_size - sizeof(struct packet_msg_prot); in goya_send_job_on_qman0()
3029 fence_pkt->ctl = cpu_to_le32(tmp); in goya_send_job_on_qman0()
3030 fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL); in goya_send_job_on_qman0()
3031 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in goya_send_job_on_qman0()
3034 job->job_cb_size, cb->bus_address); in goya_send_job_on_qman0()
3036 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); in goya_send_job_on_qman0()
3046 if (rc == -ETIMEDOUT) { in goya_send_job_on_qman0()
3047 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); in goya_send_job_on_qman0()
3052 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_send_job_on_qman0()
3063 struct goya_device *goya = hdev->asic_specific; in goya_send_cpu_message()
3065 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) { in goya_send_cpu_message()
3089 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_test_queue()
3092 dev_err(hdev->dev, in goya_test_queue()
3095 return -ENOMEM; in goya_test_queue()
3100 fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, in goya_test_queue()
3104 dev_err(hdev->dev, in goya_test_queue()
3107 rc = -ENOMEM; in goya_test_queue()
3114 fence_pkt->ctl = cpu_to_le32(tmp); in goya_test_queue()
3115 fence_pkt->value = cpu_to_le32(fence_val); in goya_test_queue()
3116 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in goya_test_queue()
3122 dev_err(hdev->dev, in goya_test_queue()
3133 if (rc == -ETIMEDOUT) { in goya_test_queue()
3134 dev_err(hdev->dev, in goya_test_queue()
3137 rc = -EIO; in goya_test_queue()
3141 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt, in goya_test_queue()
3144 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_test_queue()
3151 struct goya_device *goya = hdev->asic_specific; in goya_test_cpu_queue()
3157 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_test_cpu_queue()
3170 ret_val = -EINVAL; in goya_test_queues()
3184 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in goya_dma_pool_zalloc()
3197 dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE; in goya_dma_pool_free()
3199 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); in goya_dma_pool_free()
3208 *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + in goya_cpu_accessible_dma_pool_alloc()
3226 if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir)) in goya_dma_map_sg()
3227 return -ENOMEM; in goya_dma_map_sg()
3231 sg->dma_address += HOST_PHYS_BASE; in goya_dma_map_sg()
3244 sg->dma_address -= HOST_PHYS_BASE; in goya_dma_unmap_sg()
3246 dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir); in goya_dma_unmap_sg()
3258 for_each_sg(sgt->sgl, sg, sgt->nents, count) { in goya_get_dma_desc_list_size()
3266 while ((count + 1) < sgt->nents) { in goya_get_dma_desc_list_size()
3298 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3299 parser->job_userptr_list, &userptr)) in goya_pin_memory_before_cs()
3304 return -ENOMEM; in goya_pin_memory_before_cs()
3306 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3311 list_add_tail(&userptr->job_node, parser->job_userptr_list); in goya_pin_memory_before_cs()
3313 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl, in goya_pin_memory_before_cs()
3314 userptr->sgt->nents, dir); in goya_pin_memory_before_cs()
3316 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in goya_pin_memory_before_cs()
3320 userptr->dma_mapped = true; in goya_pin_memory_before_cs()
3321 userptr->dir = dir; in goya_pin_memory_before_cs()
3324 parser->patched_cb_size += in goya_pin_memory_before_cs()
3325 goya_get_dma_desc_list_size(hdev, userptr->sgt); in goya_pin_memory_before_cs()
3330 list_del(&userptr->job_node); in goya_pin_memory_before_cs()
3350 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_host()
3360 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n"); in goya_validate_dma_pkt_host()
3363 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3364 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3370 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3373 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3374 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3378 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n"); in goya_validate_dma_pkt_host()
3380 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3381 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3387 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3389 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3390 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3393 dev_err(hdev->dev, "DMA direction is undefined\n"); in goya_validate_dma_pkt_host()
3394 return -EFAULT; in goya_validate_dma_pkt_host()
3399 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_host()
3400 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_host()
3401 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_host()
3403 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3406 user_dma_pkt->tsize); in goya_validate_dma_pkt_host()
3407 return -EFAULT; in goya_validate_dma_pkt_host()
3411 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_host()
3412 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_host()
3413 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_host()
3415 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3418 user_dma_pkt->tsize); in goya_validate_dma_pkt_host()
3419 return -EFAULT; in goya_validate_dma_pkt_host()
3424 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_host()
3427 (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) { in goya_validate_dma_pkt_host()
3428 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3430 return -EFAULT; in goya_validate_dma_pkt_host()
3448 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_no_host()
3453 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n"); in goya_validate_dma_pkt_no_host()
3454 dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_no_host()
3455 sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_no_host()
3457 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n"); in goya_validate_dma_pkt_no_host()
3458 sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_no_host()
3459 dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_no_host()
3463 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_no_host()
3464 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_no_host()
3465 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_no_host()
3466 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3467 sram_memory_addr, user_dma_pkt->tsize); in goya_validate_dma_pkt_no_host()
3468 return -EFAULT; in goya_validate_dma_pkt_no_host()
3472 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_no_host()
3473 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_no_host()
3474 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_no_host()
3475 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3476 dram_memory_addr, user_dma_pkt->tsize); in goya_validate_dma_pkt_no_host()
3477 return -EFAULT; in goya_validate_dma_pkt_no_host()
3480 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_no_host()
3493 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_no_mmu()
3494 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3495 le64_to_cpu(user_dma_pkt->src_addr)); in goya_validate_dma_pkt_no_mmu()
3496 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3497 le64_to_cpu(user_dma_pkt->dst_addr)); in goya_validate_dma_pkt_no_mmu()
3498 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_no_mmu()
3500 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_no_mmu()
3508 if (user_dma_pkt->tsize == 0) { in goya_validate_dma_pkt_no_mmu()
3509 dev_err(hdev->dev, in goya_validate_dma_pkt_no_mmu()
3511 return -EINVAL; in goya_validate_dma_pkt_no_mmu()
3526 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_mmu()
3527 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3528 le64_to_cpu(user_dma_pkt->src_addr)); in goya_validate_dma_pkt_mmu()
3529 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3530 le64_to_cpu(user_dma_pkt->dst_addr)); in goya_validate_dma_pkt_mmu()
3531 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_mmu()
3534 * WA for HW-23. in goya_validate_dma_pkt_mmu()
3538 if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 && in goya_validate_dma_pkt_mmu()
3539 hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr), in goya_validate_dma_pkt_mmu()
3540 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_mmu()
3541 hdev->asic_prop.pmmu.start_addr, in goya_validate_dma_pkt_mmu()
3542 hdev->asic_prop.pmmu.end_addr)) { in goya_validate_dma_pkt_mmu()
3543 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3545 return -EFAULT; in goya_validate_dma_pkt_mmu()
3548 if (user_dma_pkt->tsize == 0) { in goya_validate_dma_pkt_mmu()
3549 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3551 return -EINVAL; in goya_validate_dma_pkt_mmu()
3554 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_mmu()
3563 struct goya_device *goya = hdev->asic_specific; in goya_validate_wreg32()
3567 reg_offset = le32_to_cpu(wreg_pkt->ctl) & in goya_validate_wreg32()
3570 dev_dbg(hdev->dev, "WREG32 packet details:\n"); in goya_validate_wreg32()
3571 dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset); in goya_validate_wreg32()
3572 dev_dbg(hdev->dev, "value == 0x%x\n", in goya_validate_wreg32()
3573 le32_to_cpu(wreg_pkt->value)); in goya_validate_wreg32()
3576 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n", in goya_validate_wreg32()
3578 return -EPERM; in goya_validate_wreg32()
3584 * non-secured property in goya_validate_wreg32()
3586 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_validate_wreg32()
3592 if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) || in goya_validate_wreg32()
3593 (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) { in goya_validate_wreg32()
3595 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n", in goya_validate_wreg32()
3596 wreg_pkt->value); in goya_validate_wreg32()
3597 return -EPERM; in goya_validate_wreg32()
3609 parser->patched_cb_size = 0; in goya_validate_cb()
3612 while (cb_parsed_length < parser->user_cb_size) { in goya_validate_cb()
3617 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in goya_validate_cb()
3620 (le64_to_cpu(user_pkt->header) & in goya_validate_cb()
3625 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_validate_cb()
3626 rc = -EINVAL; in goya_validate_cb()
3632 if (cb_parsed_length > parser->user_cb_size) { in goya_validate_cb()
3633 dev_err(hdev->dev, in goya_validate_cb()
3635 rc = -EINVAL; in goya_validate_cb()
3648 parser->patched_cb_size += pkt_size; in goya_validate_cb()
3652 dev_err(hdev->dev, in goya_validate_cb()
3654 rc = -EPERM; in goya_validate_cb()
3658 dev_err(hdev->dev, in goya_validate_cb()
3660 rc = -EPERM; in goya_validate_cb()
3664 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_validate_cb()
3665 rc = -EPERM; in goya_validate_cb()
3669 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_validate_cb()
3670 rc = -EPERM; in goya_validate_cb()
3686 parser->patched_cb_size += pkt_size; in goya_validate_cb()
3690 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_validate_cb()
3692 rc = -EINVAL; in goya_validate_cb()
3703 * 2. A packet that will generate MSI-X interrupt in goya_validate_cb()
3705 parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; in goya_validate_cb()
3729 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_patch_dma_packet()
3738 (user_dma_pkt->tsize == 0)) { in goya_patch_dma_packet()
3745 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_patch_dma_packet()
3746 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_patch_dma_packet()
3751 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_patch_dma_packet()
3752 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_patch_dma_packet()
3758 le32_to_cpu(user_dma_pkt->tsize), in goya_patch_dma_packet()
3759 parser->job_userptr_list, &userptr) == false)) { in goya_patch_dma_packet()
3760 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", in goya_patch_dma_packet()
3761 addr, user_dma_pkt->tsize); in goya_patch_dma_packet()
3762 return -EFAULT; in goya_patch_dma_packet()
3775 sgt = userptr->sgt; in goya_patch_dma_packet()
3778 for_each_sg(sgt->sgl, sg, sgt->nents, count) { in goya_patch_dma_packet()
3785 while ((count + 1) < sgt->nents) { in goya_patch_dma_packet()
3803 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_patch_dma_packet()
3808 new_dma_pkt->ctl = cpu_to_le32(ctl); in goya_patch_dma_packet()
3809 new_dma_pkt->tsize = cpu_to_le32((u32) len); in goya_patch_dma_packet()
3812 new_dma_pkt->src_addr = cpu_to_le64(dma_addr); in goya_patch_dma_packet()
3813 new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr); in goya_patch_dma_packet()
3815 new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr); in goya_patch_dma_packet()
3816 new_dma_pkt->dst_addr = cpu_to_le64(dma_addr); in goya_patch_dma_packet()
3826 dev_err(hdev->dev, in goya_patch_dma_packet()
3828 return -EFAULT; in goya_patch_dma_packet()
3831 /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */ in goya_patch_dma_packet()
3832 new_dma_pkt--; in goya_patch_dma_packet()
3833 new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask); in goya_patch_dma_packet()
3848 while (cb_parsed_length < parser->user_cb_size) { in goya_patch_cb()
3854 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in goya_patch_cb()
3855 kernel_pkt = parser->patched_cb->kernel_address + in goya_patch_cb()
3859 (le64_to_cpu(user_pkt->header) & in goya_patch_cb()
3864 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_patch_cb()
3865 rc = -EINVAL; in goya_patch_cb()
3871 if (cb_parsed_length > parser->user_cb_size) { in goya_patch_cb()
3872 dev_err(hdev->dev, in goya_patch_cb()
3874 rc = -EINVAL; in goya_patch_cb()
3895 dev_err(hdev->dev, in goya_patch_cb()
3897 rc = -EPERM; in goya_patch_cb()
3901 dev_err(hdev->dev, in goya_patch_cb()
3903 rc = -EPERM; in goya_patch_cb()
3907 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_patch_cb()
3908 rc = -EPERM; in goya_patch_cb()
3912 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_patch_cb()
3913 rc = -EPERM; in goya_patch_cb()
3925 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_patch_cb()
3927 rc = -EINVAL; in goya_patch_cb()
3949 * 2. A packet that will generate MSI-X interrupt in goya_parse_cb_mmu()
3951 parser->patched_cb_size = parser->user_cb_size + in goya_parse_cb_mmu()
3954 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, in goya_parse_cb_mmu()
3955 parser->patched_cb_size, false, false, in goya_parse_cb_mmu()
3959 dev_err(hdev->dev, in goya_parse_cb_mmu()
3966 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3969 if (!parser->patched_cb) { in goya_parse_cb_mmu()
3970 dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", in goya_parse_cb_mmu()
3972 rc = -EFAULT; in goya_parse_cb_mmu()
3977 * The check that parser->user_cb_size <= parser->user_cb->size was done in goya_parse_cb_mmu()
3980 memcpy(parser->patched_cb->kernel_address, in goya_parse_cb_mmu()
3981 parser->user_cb->kernel_address, in goya_parse_cb_mmu()
3982 parser->user_cb_size); in goya_parse_cb_mmu()
3984 patched_cb_size = parser->patched_cb_size; in goya_parse_cb_mmu()
3987 user_cb = parser->user_cb; in goya_parse_cb_mmu()
3988 parser->user_cb = parser->patched_cb; in goya_parse_cb_mmu()
3990 parser->user_cb = user_cb; in goya_parse_cb_mmu()
3993 hl_cb_put(parser->patched_cb); in goya_parse_cb_mmu()
3997 if (patched_cb_size != parser->patched_cb_size) { in goya_parse_cb_mmu()
3998 dev_err(hdev->dev, "user CB size mismatch\n"); in goya_parse_cb_mmu()
3999 hl_cb_put(parser->patched_cb); in goya_parse_cb_mmu()
4000 rc = -EINVAL; in goya_parse_cb_mmu()
4011 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
4028 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, in goya_parse_cb_no_mmu()
4029 parser->patched_cb_size, false, false, in goya_parse_cb_no_mmu()
4032 dev_err(hdev->dev, in goya_parse_cb_no_mmu()
4038 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
4041 if (!parser->patched_cb) { in goya_parse_cb_no_mmu()
4042 dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", in goya_parse_cb_no_mmu()
4044 rc = -EFAULT; in goya_parse_cb_no_mmu()
4051 hl_cb_put(parser->patched_cb); in goya_parse_cb_no_mmu()
4060 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
4065 hl_userptr_delete_list(hdev, parser->job_userptr_list); in goya_parse_cb_no_mmu()
4072 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in goya_parse_cb_no_ext_queue()
4073 struct goya_device *goya = hdev->asic_specific; in goya_parse_cb_no_ext_queue()
4075 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_parse_cb_no_ext_queue()
4080 (u64) (uintptr_t) parser->user_cb, in goya_parse_cb_no_ext_queue()
4081 parser->user_cb_size, in goya_parse_cb_no_ext_queue()
4082 asic_prop->sram_user_base_address, in goya_parse_cb_no_ext_queue()
4083 asic_prop->sram_end_address)) in goya_parse_cb_no_ext_queue()
4087 (u64) (uintptr_t) parser->user_cb, in goya_parse_cb_no_ext_queue()
4088 parser->user_cb_size, in goya_parse_cb_no_ext_queue()
4089 asic_prop->dram_user_base_address, in goya_parse_cb_no_ext_queue()
4090 asic_prop->dram_end_address)) in goya_parse_cb_no_ext_queue()
4093 dev_err(hdev->dev, in goya_parse_cb_no_ext_queue()
4095 parser->user_cb, parser->user_cb_size); in goya_parse_cb_no_ext_queue()
4097 return -EFAULT; in goya_parse_cb_no_ext_queue()
4102 struct goya_device *goya = hdev->asic_specific; in goya_cs_parser()
4104 if (parser->queue_type == QUEUE_TYPE_INT) in goya_cs_parser()
4107 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_cs_parser()
4120 cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2); in goya_add_end_of_cb_packets()
4125 cq_pkt->ctl = cpu_to_le32(tmp); in goya_add_end_of_cb_packets()
4126 cq_pkt->value = cpu_to_le32(cq_val); in goya_add_end_of_cb_packets()
4127 cq_pkt->addr = cpu_to_le64(cq_addr); in goya_add_end_of_cb_packets()
4133 cq_pkt->ctl = cpu_to_le32(tmp); in goya_add_end_of_cb_packets()
4134 cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF); in goya_add_end_of_cb_packets()
4135 cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF); in goya_add_end_of_cb_packets()
4153 ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4); in goya_clear_sm_regs()
4156 ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4); in goya_clear_sm_regs()
4169 * goya_debugfs_read32 - read a 32bit value from a given device or a host mapped
4186 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_read32()
4193 *val = RREG32(addr - CFG_BASE); in goya_debugfs_read32()
4198 *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_read32()
4199 (addr - SRAM_BASE_ADDR)); in goya_debugfs_read32()
4201 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) { in goya_debugfs_read32()
4204 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_read32()
4208 *val = readl(hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_read32()
4209 (addr - bar_base_addr)); in goya_debugfs_read32()
4215 rc = -EIO; in goya_debugfs_read32()
4219 *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE); in goya_debugfs_read32()
4222 rc = -EFAULT; in goya_debugfs_read32()
4229 * goya_debugfs_write32 - write a 32bit value to a given device or a host mapped
4246 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_write32()
4253 WREG32(addr - CFG_BASE, val); in goya_debugfs_write32()
4258 writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_write32()
4259 (addr - SRAM_BASE_ADDR)); in goya_debugfs_write32()
4261 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) { in goya_debugfs_write32()
4264 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_write32()
4268 writel(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_write32()
4269 (addr - bar_base_addr)); in goya_debugfs_write32()
4275 rc = -EIO; in goya_debugfs_write32()
4279 *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; in goya_debugfs_write32()
4282 rc = -EFAULT; in goya_debugfs_write32()
4291 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_read64()
4297 if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) { in goya_debugfs_read64()
4298 u32 val_l = RREG32(addr - CFG_BASE); in goya_debugfs_read64()
4299 u32 val_h = RREG32(addr + sizeof(u32) - CFG_BASE); in goya_debugfs_read64()
4304 (addr <= SRAM_BASE_ADDR + SRAM_SIZE - sizeof(u64))) { in goya_debugfs_read64()
4306 *val = readq(hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_read64()
4307 (addr - SRAM_BASE_ADDR)); in goya_debugfs_read64()
4310 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) { in goya_debugfs_read64()
4313 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_read64()
4317 *val = readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_read64()
4318 (addr - bar_base_addr)); in goya_debugfs_read64()
4324 rc = -EIO; in goya_debugfs_read64()
4328 *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE); in goya_debugfs_read64()
4331 rc = -EFAULT; in goya_debugfs_read64()
4340 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_write64()
4346 if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) { in goya_debugfs_write64()
4347 WREG32(addr - CFG_BASE, lower_32_bits(val)); in goya_debugfs_write64()
4348 WREG32(addr + sizeof(u32) - CFG_BASE, upper_32_bits(val)); in goya_debugfs_write64()
4351 (addr <= SRAM_BASE_ADDR + SRAM_SIZE - sizeof(u64))) { in goya_debugfs_write64()
4353 writeq(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_write64()
4354 (addr - SRAM_BASE_ADDR)); in goya_debugfs_write64()
4357 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) { in goya_debugfs_write64()
4360 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_write64()
4364 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_write64()
4365 (addr - bar_base_addr)); in goya_debugfs_write64()
4371 rc = -EIO; in goya_debugfs_write64()
4375 *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; in goya_debugfs_write64()
4378 rc = -EFAULT; in goya_debugfs_write64()
4387 dev_err(hdev->dev, "Reading via DMA is unimplemented yet\n"); in goya_debugfs_read_dma()
4388 return -EPERM; in goya_debugfs_read_dma()
4393 struct goya_device *goya = hdev->asic_specific; in goya_read_pte()
4395 if (hdev->hard_reset_pending) in goya_read_pte()
4398 return readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_read_pte()
4399 (addr - goya->ddr_bar_cur_addr)); in goya_read_pte()
4404 struct goya_device *goya = hdev->asic_specific; in goya_write_pte()
4406 if (hdev->hard_reset_pending) in goya_write_pte()
4409 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_write_pte()
4410 (addr - goya->ddr_bar_cur_addr)); in goya_write_pte()
4538 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3; in goya_get_event_desc()
4542 index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0; in goya_get_event_desc()
4546 index = event_type - GOYA_ASYNC_EVENT_ID_PLL0; in goya_get_event_desc()
4557 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_DEC) / 3; in goya_get_event_desc()
4568 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR) / 10; in goya_get_event_desc()
4572 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_CMDQ; in goya_get_event_desc()
4576 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_QM; in goya_get_event_desc()
4580 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_QM; in goya_get_event_desc()
4584 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH; in goya_get_event_desc()
4595 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10; in goya_get_event_desc()
4599 index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0; in goya_get_event_desc()
4614 dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n"); in goya_print_razwi_info()
4619 dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n"); in goya_print_razwi_info()
4624 dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n"); in goya_print_razwi_info()
4629 dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n"); in goya_print_razwi_info()
4636 struct goya_device *goya = hdev->asic_specific; in goya_print_mmu_error_info()
4640 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_print_mmu_error_info()
4649 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", in goya_print_mmu_error_info()
4659 struct hl_hw_queue *q = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_print_out_of_sync_info()
4661 dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%u\n", in goya_print_out_of_sync_info()
4662 sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci)); in goya_print_out_of_sync_info()
4671 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in goya_print_irq_info()
4693 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */ in goya_unmask_irq_arr()
4698 dev_err(hdev->dev, "too many elements in IRQ array\n"); in goya_unmask_irq_arr()
4699 return -EINVAL; in goya_unmask_irq_arr()
4704 return -ENOMEM; in goya_unmask_irq_arr()
4707 pkt->length = cpu_to_le32(irq_num_entries); in goya_unmask_irq_arr()
4712 for (irq_arr_index = 0, goya_irq_arr = (__le32 *) &pkt->irqs; in goya_unmask_irq_arr()
4717 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY << in goya_unmask_irq_arr()
4720 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt, in goya_unmask_irq_arr()
4724 dev_err(hdev->dev, "failed to unmask IRQ array\n"); in goya_unmask_irq_arr()
4753 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), in goya_unmask_irq()
4757 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type); in goya_unmask_irq()
4766 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4767 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4771 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4772 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4776 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4777 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4781 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4782 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4787 dev_err(hdev->dev, "Received invalid clock change event %d\n", in goya_print_clk_change_info()
4795 u32 ctl = le32_to_cpu(eq_entry->hdr.ctl); in goya_handle_eqe()
4798 struct goya_device *goya = hdev->asic_specific; in goya_handle_eqe()
4801 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in goya_handle_eqe()
4802 event_type, GOYA_ASYNC_EVENT_ID_SIZE - 1); in goya_handle_eqe()
4806 goya->events_stat[event_type]++; in goya_handle_eqe()
4807 goya->events_stat_aggregate[event_type]++; in goya_handle_eqe()
4834 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4893 goya_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); in goya_handle_eqe()
4894 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4901 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", in goya_handle_eqe()
4909 struct goya_device *goya = hdev->asic_specific; in goya_get_events_stat()
4912 *size = (u32) sizeof(goya->events_stat_aggregate); in goya_get_events_stat()
4913 return goya->events_stat_aggregate; in goya_get_events_stat()
4916 *size = (u32) sizeof(goya->events_stat); in goya_get_events_stat()
4917 return goya->events_stat; in goya_get_events_stat()
4934 return -ENOMEM; in goya_memset_device_memory()
4936 lin_dma_pkt = cb->kernel_address; in goya_memset_device_memory()
4948 lin_dma_pkt->ctl = cpu_to_le32(ctl); in goya_memset_device_memory()
4950 lin_dma_pkt->src_addr = cpu_to_le64(val); in goya_memset_device_memory()
4951 lin_dma_pkt->dst_addr = cpu_to_le64(addr); in goya_memset_device_memory()
4953 lin_dma_pkt->tsize = cpu_to_le32(SZ_2G); in goya_memset_device_memory()
4955 lin_dma_pkt->tsize = cpu_to_le32(size); in goya_memset_device_memory()
4957 size -= SZ_2G; in goya_memset_device_memory()
4960 } while (--lin_dma_pkts_cnt); in goya_memset_device_memory()
4964 dev_err(hdev->dev, "Failed to allocate a new job\n"); in goya_memset_device_memory()
4965 rc = -ENOMEM; in goya_memset_device_memory()
4969 job->id = 0; in goya_memset_device_memory()
4970 job->user_cb = cb; in goya_memset_device_memory()
4971 atomic_inc(&job->user_cb->cs_cnt); in goya_memset_device_memory()
4972 job->user_cb_size = cb_size; in goya_memset_device_memory()
4973 job->hw_queue_id = GOYA_QUEUE_ID_DMA_0; in goya_memset_device_memory()
4974 job->patched_cb = job->user_cb; in goya_memset_device_memory()
4975 job->job_cb_size = job->user_cb_size; in goya_memset_device_memory()
4983 atomic_dec(&cb->cs_cnt); in goya_memset_device_memory()
4987 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); in goya_memset_device_memory()
4994 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_context_switch()
4995 u64 addr = prop->sram_base_address, sob_addr; in goya_context_switch()
4996 u32 size = hdev->pldm ? 0x10000 : prop->sram_size; in goya_context_switch()
4999 u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO - in goya_context_switch()
5004 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n"); in goya_context_switch()
5014 (dma_id - 1) * 4; in goya_context_switch()
5028 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_clear_pgt_range()
5029 struct goya_device *goya = hdev->asic_specific; in goya_mmu_clear_pgt_range()
5030 u64 addr = prop->mmu_pgt_addr; in goya_mmu_clear_pgt_range()
5031 u32 size = prop->mmu_pgt_size + MMU_DRAM_DEFAULT_PAGE_SIZE + in goya_mmu_clear_pgt_range()
5034 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_clear_pgt_range()
5042 struct goya_device *goya = hdev->asic_specific; in goya_mmu_set_dram_default_page()
5043 u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; in goya_mmu_set_dram_default_page()
5047 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_set_dram_default_page()
5055 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_add_mappings_for_device_cpu()
5056 struct goya_device *goya = hdev->asic_specific; in goya_mmu_add_mappings_for_device_cpu()
5060 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_add_mappings_for_device_cpu()
5064 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
5065 prop->dram_base_address + off, in goya_mmu_add_mappings_for_device_cpu()
5066 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_add_mappings_for_device_cpu()
5069 dev_err(hdev->dev, "Map failed for address 0x%llx\n", in goya_mmu_add_mappings_for_device_cpu()
5070 prop->dram_base_address + off); in goya_mmu_add_mappings_for_device_cpu()
5075 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_add_mappings_for_device_cpu()
5076 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
5078 hdev->cpu_accessible_dma_address, in goya_mmu_add_mappings_for_device_cpu()
5082 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
5084 off -= PAGE_SIZE_2MB; in goya_mmu_add_mappings_for_device_cpu()
5089 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
5091 hdev->cpu_accessible_dma_address + cpu_off, in goya_mmu_add_mappings_for_device_cpu()
5094 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
5096 cpu_off -= PAGE_SIZE_4KB; in goya_mmu_add_mappings_for_device_cpu()
5110 goya->device_cpu_mmu_mappings_done = true; in goya_mmu_add_mappings_for_device_cpu()
5115 for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB) in goya_mmu_add_mappings_for_device_cpu()
5116 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
5119 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
5123 for (; off >= 0 ; off -= PAGE_SIZE_2MB) in goya_mmu_add_mappings_for_device_cpu()
5124 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
5125 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_add_mappings_for_device_cpu()
5127 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
5129 prop->dram_base_address + off); in goya_mmu_add_mappings_for_device_cpu()
5136 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_remove_device_cpu_mappings()
5137 struct goya_device *goya = hdev->asic_specific; in goya_mmu_remove_device_cpu_mappings()
5140 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_remove_device_cpu_mappings()
5143 if (!goya->device_cpu_mmu_mappings_done) in goya_mmu_remove_device_cpu_mappings()
5149 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_remove_device_cpu_mappings()
5150 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
5153 dev_warn(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
5157 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
5161 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
5167 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
5168 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_remove_device_cpu_mappings()
5170 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
5172 prop->dram_base_address + off); in goya_mmu_remove_device_cpu_mappings()
5174 goya->device_cpu_mmu_mappings_done = false; in goya_mmu_remove_device_cpu_mappings()
5179 struct goya_device *goya = hdev->asic_specific; in goya_mmu_prepare()
5182 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_prepare()
5186 dev_crit(hdev->dev, "asid %u is too big\n", asid); in goya_mmu_prepare()
5198 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache()
5202 if (!(goya->hw_cap_initialized & HW_CAP_MMU) || in goya_mmu_invalidate_cache()
5203 hdev->hard_reset_pending) in goya_mmu_invalidate_cache()
5210 if (hdev->pldm) in goya_mmu_invalidate_cache()
5227 dev_err_ratelimited(hdev->dev, in goya_mmu_invalidate_cache()
5242 return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); in goya_mmu_invalidate_cache_range()
5247 struct goya_device *goya = hdev->asic_specific; in goya_send_heartbeat()
5249 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_send_heartbeat()
5257 struct goya_device *goya = hdev->asic_specific; in goya_cpucp_info_get()
5258 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_cpucp_info_get()
5262 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_cpucp_info_get()
5271 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); in goya_cpucp_info_get()
5275 dev_err(hdev->dev, in goya_cpucp_info_get()
5281 prop->dram_size = dram_size; in goya_cpucp_info_get()
5282 prop->dram_end_address = prop->dram_base_address + dram_size; in goya_cpucp_info_get()
5285 if (!strlen(prop->cpucp_info.card_name)) in goya_cpucp_info_get()
5286 strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, in goya_cpucp_info_get()
5305 const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; in goya_is_device_idle()
5306 const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; in goya_is_device_idle()
5316 "--- ------- ------------ -------------\n"); in goya_is_device_idle()
5318 offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0; in goya_is_device_idle()
5337 "--- ------- ------------ -------------- ----------\n"); in goya_is_device_idle()
5339 offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0; in goya_is_device_idle()
5360 "--- ------- ------------ -------------- -----------\n"); in goya_is_device_idle()
5382 __acquires(&goya->hw_queues_lock) in goya_hw_queues_lock()
5384 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_lock()
5386 spin_lock(&goya->hw_queues_lock); in goya_hw_queues_lock()
5390 __releases(&goya->hw_queues_lock) in goya_hw_queues_unlock()
5392 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_unlock()
5394 spin_unlock(&goya->hw_queues_lock); in goya_hw_queues_unlock()
5399 return hdev->pdev->device; in goya_get_pci_id()
5405 struct goya_device *goya = hdev->asic_specific; in goya_get_eeprom_data()
5407 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_get_eeprom_data()
5420 if (ctx->asid != HL_KERNEL_ASID_ID) in goya_ctx_init()
5421 goya_mmu_prepare(ctx->hdev, ctx->asid); in goya_ctx_init()
5467 dev_dbg(hdev->dev, "Working in 64-bit DMA mode\n"); in goya_set_dma_mask_from_fw()
5468 hdev->power9_64bit_dma_enable = 1; in goya_set_dma_mask_from_fw()
5469 hdev->dma_mask = 64; in goya_set_dma_mask_from_fw()
5471 dev_dbg(hdev->dev, "Working in 48-bit DMA mode\n"); in goya_set_dma_mask_from_fw()
5472 hdev->power9_64bit_dma_enable = 0; in goya_set_dma_mask_from_fw()
5473 hdev->dma_mask = 48; in goya_set_dma_mask_from_fw()
5493 return -EINVAL; in goya_collective_wait_create_jobs()
5504 return -EPERM; in goya_get_hw_block_id()
5510 return -EPERM; in goya_block_mmap()
5529 default: return -EINVAL; in goya_map_pll_idx_to_fw_idx()
5575 hdev->state_dump_specs.props = goya_state_dump_specs_props; in goya_state_dump_init()
5576 hdev->state_dump_specs.funcs = goya_state_dump_funcs; in goya_state_dump_init()
5682 * goya_set_asic_funcs - set Goya function pointers
5689 hdev->asic_funcs = &goya_funcs; in goya_set_asic_funcs()