Lines Matching refs:hdev
327 static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
328 static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
329 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
330 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
332 void goya_get_fixed_properties(struct hl_device *hdev) in goya_get_fixed_properties() argument
334 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_get_fixed_properties()
371 if (hdev->pldm) in goya_get_fixed_properties()
407 static int goya_pci_bars_map(struct hl_device *hdev) in goya_pci_bars_map() argument
413 rc = hl_pci_bars_map(hdev, name, is_wc); in goya_pci_bars_map()
417 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_pci_bars_map()
423 static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr) in goya_set_ddr_bar_base() argument
425 struct goya_device *goya = hdev->asic_specific; in goya_set_ddr_bar_base()
433 rc = hl_pci_set_dram_bar_base(hdev, 1, 4, addr); in goya_set_ddr_bar_base()
453 static int goya_init_iatu(struct hl_device *hdev) in goya_init_iatu() argument
455 return hl_pci_init_iatu(hdev, SRAM_BASE_ADDR, DRAM_PHYS_BASE, in goya_init_iatu()
470 static int goya_early_init(struct hl_device *hdev) in goya_early_init() argument
472 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_early_init()
473 struct pci_dev *pdev = hdev->pdev; in goya_early_init()
477 goya_get_fixed_properties(hdev); in goya_early_init()
481 dev_err(hdev->dev, in goya_early_init()
491 dev_err(hdev->dev, in goya_early_init()
502 rc = hl_pci_init(hdev, 48); in goya_early_init()
506 if (!hdev->pldm) { in goya_early_init()
509 dev_warn(hdev->dev, in goya_early_init()
524 static int goya_early_fini(struct hl_device *hdev) in goya_early_fini() argument
526 hl_pci_fini(hdev); in goya_early_fini()
531 static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid) in goya_mmu_prepare_reg() argument
538 static void goya_qman0_set_security(struct hl_device *hdev, bool secure) in goya_qman0_set_security() argument
540 struct goya_device *goya = hdev->asic_specific; in goya_qman0_set_security()
559 static void goya_fetch_psoc_frequency(struct hl_device *hdev) in goya_fetch_psoc_frequency() argument
561 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_fetch_psoc_frequency()
569 int goya_late_init(struct hl_device *hdev) in goya_late_init() argument
571 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_late_init()
574 goya_fetch_psoc_frequency(hdev); in goya_late_init()
576 rc = goya_mmu_clear_pgt_range(hdev); in goya_late_init()
578 dev_err(hdev->dev, in goya_late_init()
583 rc = goya_mmu_set_dram_default_page(hdev); in goya_late_init()
585 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); in goya_late_init()
589 rc = goya_mmu_add_mappings_for_device_cpu(hdev); in goya_late_init()
593 rc = goya_init_cpu_queues(hdev); in goya_late_init()
597 rc = goya_test_cpu_queue(hdev); in goya_late_init()
601 rc = goya_armcp_info_get(hdev); in goya_late_init()
603 dev_err(hdev->dev, "Failed to get armcp info %d\n", rc); in goya_late_init()
613 rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS); in goya_late_init()
615 dev_err(hdev->dev, in goya_late_init()
633 void goya_late_fini(struct hl_device *hdev) in goya_late_fini() argument
638 if (!hdev->hl_chip_info->info) in goya_late_fini()
641 channel_info_arr = hdev->hl_chip_info->info; in goya_late_fini()
651 hdev->hl_chip_info->info = NULL; in goya_late_fini()
660 static int goya_sw_init(struct hl_device *hdev) in goya_sw_init() argument
677 hdev->asic_specific = goya; in goya_sw_init()
680 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), in goya_sw_init()
681 &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0); in goya_sw_init()
682 if (!hdev->dma_pool) { in goya_sw_init()
683 dev_err(hdev->dev, "failed to create DMA pool\n"); in goya_sw_init()
688 hdev->cpu_accessible_dma_mem = in goya_sw_init()
689 hdev->asic_funcs->asic_dma_alloc_coherent(hdev, in goya_sw_init()
691 &hdev->cpu_accessible_dma_address, in goya_sw_init()
694 if (!hdev->cpu_accessible_dma_mem) { in goya_sw_init()
699 dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n", in goya_sw_init()
700 &hdev->cpu_accessible_dma_address); in goya_sw_init()
702 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in goya_sw_init()
703 if (!hdev->cpu_accessible_dma_pool) { in goya_sw_init()
704 dev_err(hdev->dev, in goya_sw_init()
710 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, in goya_sw_init()
711 (uintptr_t) hdev->cpu_accessible_dma_mem, in goya_sw_init()
714 dev_err(hdev->dev, in goya_sw_init()
725 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_init()
727 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_init()
729 hdev->cpu_accessible_dma_mem, in goya_sw_init()
730 hdev->cpu_accessible_dma_address); in goya_sw_init()
732 dma_pool_destroy(hdev->dma_pool); in goya_sw_init()
745 static int goya_sw_fini(struct hl_device *hdev) in goya_sw_fini() argument
747 struct goya_device *goya = hdev->asic_specific; in goya_sw_fini()
749 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_fini()
751 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_fini()
753 hdev->cpu_accessible_dma_mem, in goya_sw_fini()
754 hdev->cpu_accessible_dma_address); in goya_sw_fini()
756 dma_pool_destroy(hdev->dma_pool); in goya_sw_fini()
763 static void goya_init_dma_qman(struct hl_device *hdev, int dma_id, in goya_init_dma_qman() argument
766 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qman()
811 static void goya_init_dma_ch(struct hl_device *hdev, int dma_id) in goya_init_dma_ch() argument
845 void goya_init_dma_qmans(struct hl_device *hdev) in goya_init_dma_qmans() argument
847 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qmans()
854 q = &hdev->kernel_queues[0]; in goya_init_dma_qmans()
857 goya_init_dma_qman(hdev, i, q->bus_address); in goya_init_dma_qmans()
858 goya_init_dma_ch(hdev, i); in goya_init_dma_qmans()
870 static void goya_disable_external_queues(struct hl_device *hdev) in goya_disable_external_queues() argument
879 static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg, in goya_stop_queue() argument
892 hdev, in goya_stop_queue()
905 hdev, in goya_stop_queue()
913 dev_err(hdev->dev, in goya_stop_queue()
929 static int goya_stop_external_queues(struct hl_device *hdev) in goya_stop_external_queues() argument
933 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
939 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n"); in goya_stop_external_queues()
943 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
949 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n"); in goya_stop_external_queues()
953 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
959 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n"); in goya_stop_external_queues()
963 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
969 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n"); in goya_stop_external_queues()
973 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
979 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n"); in goya_stop_external_queues()
994 int goya_init_cpu_queues(struct hl_device *hdev) in goya_init_cpu_queues() argument
996 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu_queues()
999 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_init_cpu_queues()
1002 if (!hdev->cpu_queues_enable) in goya_init_cpu_queues()
1008 eq = &hdev->event_queue; in goya_init_cpu_queues()
1036 hdev, in goya_init_cpu_queues()
1044 dev_err(hdev->dev, in goya_init_cpu_queues()
1053 static void goya_set_pll_refclk(struct hl_device *hdev) in goya_set_pll_refclk() argument
1091 static void goya_disable_clk_rlx(struct hl_device *hdev) in goya_disable_clk_rlx() argument
1097 static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id) in _goya_tpc_mbist_workaround() argument
1115 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n", in _goya_tpc_mbist_workaround()
1135 hdev, in _goya_tpc_mbist_workaround()
1143 dev_err(hdev->dev, in _goya_tpc_mbist_workaround()
1162 static void goya_tpc_mbist_workaround(struct hl_device *hdev) in goya_tpc_mbist_workaround() argument
1164 struct goya_device *goya = hdev->asic_specific; in goya_tpc_mbist_workaround()
1167 if (hdev->pldm) in goya_tpc_mbist_workaround()
1176 _goya_tpc_mbist_workaround(hdev, i); in goya_tpc_mbist_workaround()
1189 static void goya_init_golden_registers(struct hl_device *hdev) in goya_init_golden_registers() argument
1191 struct goya_device *goya = hdev->asic_specific; in goya_init_golden_registers()
1481 static void goya_init_mme_qman(struct hl_device *hdev) in goya_init_mme_qman() argument
1498 qman_base_addr = hdev->asic_prop.sram_base_address + in goya_init_mme_qman()
1531 static void goya_init_mme_cmdq(struct hl_device *hdev) in goya_init_mme_cmdq() argument
1548 qman_base_addr = hdev->asic_prop.sram_base_address + in goya_init_mme_cmdq()
1571 void goya_init_mme_qmans(struct hl_device *hdev) in goya_init_mme_qmans() argument
1573 struct goya_device *goya = hdev->asic_specific; in goya_init_mme_qmans()
1585 goya_init_mme_qman(hdev); in goya_init_mme_qmans()
1586 goya_init_mme_cmdq(hdev); in goya_init_mme_qmans()
1591 static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id) in goya_init_tpc_qman() argument
1609 qman_base_addr = hdev->asic_prop.sram_base_address + base_off; in goya_init_tpc_qman()
1641 static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id) in goya_init_tpc_cmdq() argument
1678 void goya_init_tpc_qmans(struct hl_device *hdev) in goya_init_tpc_qmans() argument
1680 struct goya_device *goya = hdev->asic_specific; in goya_init_tpc_qmans()
1699 goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0); in goya_init_tpc_qmans()
1700 goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1); in goya_init_tpc_qmans()
1701 goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2); in goya_init_tpc_qmans()
1702 goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3); in goya_init_tpc_qmans()
1703 goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4); in goya_init_tpc_qmans()
1704 goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5); in goya_init_tpc_qmans()
1705 goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6); in goya_init_tpc_qmans()
1706 goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7); in goya_init_tpc_qmans()
1709 goya_init_tpc_cmdq(hdev, i); in goya_init_tpc_qmans()
1720 static void goya_disable_internal_queues(struct hl_device *hdev) in goya_disable_internal_queues() argument
1758 static int goya_stop_internal_queues(struct hl_device *hdev) in goya_stop_internal_queues() argument
1768 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1774 dev_err(hdev->dev, "failed to stop MME QMAN\n"); in goya_stop_internal_queues()
1778 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1784 dev_err(hdev->dev, "failed to stop MME CMDQ\n"); in goya_stop_internal_queues()
1788 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1794 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n"); in goya_stop_internal_queues()
1798 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1804 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n"); in goya_stop_internal_queues()
1808 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1814 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n"); in goya_stop_internal_queues()
1818 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1824 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n"); in goya_stop_internal_queues()
1828 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1834 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n"); in goya_stop_internal_queues()
1838 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1844 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n"); in goya_stop_internal_queues()
1848 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1854 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n"); in goya_stop_internal_queues()
1858 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1864 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n"); in goya_stop_internal_queues()
1868 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1874 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n"); in goya_stop_internal_queues()
1878 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1884 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n"); in goya_stop_internal_queues()
1888 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1894 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n"); in goya_stop_internal_queues()
1898 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1904 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n"); in goya_stop_internal_queues()
1908 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1914 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n"); in goya_stop_internal_queues()
1918 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1924 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n"); in goya_stop_internal_queues()
1928 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1934 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n"); in goya_stop_internal_queues()
1938 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1944 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n"); in goya_stop_internal_queues()
1951 static void goya_dma_stall(struct hl_device *hdev) in goya_dma_stall() argument
1960 static void goya_tpc_stall(struct hl_device *hdev) in goya_tpc_stall() argument
1972 static void goya_mme_stall(struct hl_device *hdev) in goya_mme_stall() argument
1977 static int goya_enable_msix(struct hl_device *hdev) in goya_enable_msix() argument
1979 struct goya_device *goya = hdev->asic_specific; in goya_enable_msix()
1980 int cq_cnt = hdev->asic_prop.completion_queues_count; in goya_enable_msix()
1986 rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES, in goya_enable_msix()
1989 dev_err(hdev->dev, in goya_enable_msix()
1996 irq = pci_irq_vector(hdev->pdev, i); in goya_enable_msix()
1998 &hdev->completion_queue[i]); in goya_enable_msix()
2000 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2005 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_enable_msix()
2009 &hdev->event_queue); in goya_enable_msix()
2011 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2020 free_irq(pci_irq_vector(hdev->pdev, i), in goya_enable_msix()
2021 &hdev->completion_queue[i]); in goya_enable_msix()
2023 pci_free_irq_vectors(hdev->pdev); in goya_enable_msix()
2027 static void goya_sync_irqs(struct hl_device *hdev) in goya_sync_irqs() argument
2029 struct goya_device *goya = hdev->asic_specific; in goya_sync_irqs()
2036 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in goya_sync_irqs()
2037 synchronize_irq(pci_irq_vector(hdev->pdev, i)); in goya_sync_irqs()
2039 synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX)); in goya_sync_irqs()
2042 static void goya_disable_msix(struct hl_device *hdev) in goya_disable_msix() argument
2044 struct goya_device *goya = hdev->asic_specific; in goya_disable_msix()
2050 goya_sync_irqs(hdev); in goya_disable_msix()
2052 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_disable_msix()
2053 free_irq(irq, &hdev->event_queue); in goya_disable_msix()
2055 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in goya_disable_msix()
2056 irq = pci_irq_vector(hdev->pdev, i); in goya_disable_msix()
2057 free_irq(irq, &hdev->completion_queue[i]); in goya_disable_msix()
2060 pci_free_irq_vectors(hdev->pdev); in goya_disable_msix()
2065 static void goya_enable_timestamp(struct hl_device *hdev) in goya_enable_timestamp() argument
2078 static void goya_disable_timestamp(struct hl_device *hdev) in goya_disable_timestamp() argument
2084 static void goya_halt_engines(struct hl_device *hdev, bool hard_reset) in goya_halt_engines() argument
2088 dev_info(hdev->dev, in goya_halt_engines()
2091 if (hdev->pldm) { in goya_halt_engines()
2110 goya_stop_external_queues(hdev); in goya_halt_engines()
2111 goya_stop_internal_queues(hdev); in goya_halt_engines()
2115 goya_dma_stall(hdev); in goya_halt_engines()
2116 goya_tpc_stall(hdev); in goya_halt_engines()
2117 goya_mme_stall(hdev); in goya_halt_engines()
2121 goya_disable_external_queues(hdev); in goya_halt_engines()
2122 goya_disable_internal_queues(hdev); in goya_halt_engines()
2124 goya_disable_timestamp(hdev); in goya_halt_engines()
2127 goya_disable_msix(hdev); in goya_halt_engines()
2128 goya_mmu_remove_device_cpu_mappings(hdev); in goya_halt_engines()
2130 goya_sync_irqs(hdev); in goya_halt_engines()
2142 static int goya_push_uboot_to_device(struct hl_device *hdev) in goya_push_uboot_to_device() argument
2148 dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET; in goya_push_uboot_to_device()
2150 return hl_fw_push_fw_to_device(hdev, fw_name, dst); in goya_push_uboot_to_device()
2161 static int goya_push_linux_to_device(struct hl_device *hdev) in goya_push_linux_to_device() argument
2167 dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET; in goya_push_linux_to_device()
2169 return hl_fw_push_fw_to_device(hdev, fw_name, dst); in goya_push_linux_to_device()
2172 static int goya_pldm_init_cpu(struct hl_device *hdev) in goya_pldm_init_cpu() argument
2178 goya_init_golden_registers(hdev); in goya_pldm_init_cpu()
2191 rc = goya_push_uboot_to_device(hdev); in goya_pldm_init_cpu()
2195 rc = goya_push_linux_to_device(hdev); in goya_pldm_init_cpu()
2219 static void goya_read_device_fw_version(struct hl_device *hdev, in goya_read_device_fw_version() argument
2229 dest = hdev->asic_prop.uboot_ver; in goya_read_device_fw_version()
2234 dest = hdev->asic_prop.preboot_ver; in goya_read_device_fw_version()
2238 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); in goya_read_device_fw_version()
2245 memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off, in goya_read_device_fw_version()
2248 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", in goya_read_device_fw_version()
2254 static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout) in goya_init_cpu() argument
2256 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu()
2260 if (!hdev->cpu_enable) in goya_init_cpu()
2270 if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { in goya_init_cpu()
2271 dev_err(hdev->dev, in goya_init_cpu()
2276 if (hdev->pldm) { in goya_init_cpu()
2277 rc = goya_pldm_init_cpu(hdev); in goya_init_cpu()
2286 hdev, in goya_init_cpu()
2295 dev_err(hdev->dev, "Error in ARM u-boot!"); in goya_init_cpu()
2298 dev_err(hdev->dev, in goya_init_cpu()
2302 dev_err(hdev->dev, in goya_init_cpu()
2306 dev_err(hdev->dev, in goya_init_cpu()
2310 dev_err(hdev->dev, in goya_init_cpu()
2314 dev_err(hdev->dev, in goya_init_cpu()
2318 dev_err(hdev->dev, in goya_init_cpu()
2322 dev_err(hdev->dev, in goya_init_cpu()
2327 dev_err(hdev->dev, in goya_init_cpu()
2332 dev_err(hdev->dev, in goya_init_cpu()
2341 goya_read_device_fw_version(hdev, FW_COMP_UBOOT); in goya_init_cpu()
2342 goya_read_device_fw_version(hdev, FW_COMP_PREBOOT); in goya_init_cpu()
2344 if (!hdev->fw_loading) { in goya_init_cpu()
2345 dev_info(hdev->dev, "Skip loading FW\n"); in goya_init_cpu()
2352 rc = goya_push_linux_to_device(hdev); in goya_init_cpu()
2359 hdev, in goya_init_cpu()
2368 dev_err(hdev->dev, in goya_init_cpu()
2371 dev_err(hdev->dev, in goya_init_cpu()
2377 dev_info(hdev->dev, "Successfully loaded firmware to device\n"); in goya_init_cpu()
2385 static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, in goya_mmu_update_asid_hop0_addr() argument
2391 if (hdev->pldm) in goya_mmu_update_asid_hop0_addr()
2401 hdev, in goya_mmu_update_asid_hop0_addr()
2409 dev_err(hdev->dev, in goya_mmu_update_asid_hop0_addr()
2417 int goya_mmu_init(struct hl_device *hdev) in goya_mmu_init() argument
2419 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_init()
2420 struct goya_device *goya = hdev->asic_specific; in goya_mmu_init()
2424 if (!hdev->mmu_enable) in goya_mmu_init()
2430 hdev->dram_supports_virtual_memory = true; in goya_mmu_init()
2431 hdev->dram_default_page_mapping = true; in goya_mmu_init()
2437 rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr); in goya_mmu_init()
2439 dev_err(hdev->dev, in goya_mmu_init()
2456 hdev->asic_funcs->mmu_invalidate_cache(hdev, true); in goya_mmu_init()
2475 static int goya_hw_init(struct hl_device *hdev) in goya_hw_init() argument
2477 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_hw_init()
2481 dev_info(hdev->dev, "Starting initialization of H/W\n"); in goya_hw_init()
2494 rc = goya_init_cpu(hdev, GOYA_CPU_TIMEOUT_USEC); in goya_hw_init()
2496 dev_err(hdev->dev, "failed to initialize CPU\n"); in goya_hw_init()
2500 goya_tpc_mbist_workaround(hdev); in goya_hw_init()
2502 goya_init_golden_registers(hdev); in goya_hw_init()
2508 if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE + in goya_hw_init()
2511 dev_err(hdev->dev, in goya_hw_init()
2516 rc = goya_mmu_init(hdev); in goya_hw_init()
2520 goya_init_security(hdev); in goya_hw_init()
2522 goya_init_dma_qmans(hdev); in goya_hw_init()
2524 goya_init_mme_qmans(hdev); in goya_hw_init()
2526 goya_init_tpc_qmans(hdev); in goya_hw_init()
2528 goya_enable_timestamp(hdev); in goya_hw_init()
2531 rc = goya_enable_msix(hdev); in goya_hw_init()
2541 goya_disable_internal_queues(hdev); in goya_hw_init()
2542 goya_disable_external_queues(hdev); in goya_hw_init()
2554 static void goya_hw_fini(struct hl_device *hdev, bool hard_reset) in goya_hw_fini() argument
2556 struct goya_device *goya = hdev->asic_specific; in goya_hw_fini()
2559 if (hdev->pldm) in goya_hw_fini()
2565 goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE); in goya_hw_fini()
2566 goya_disable_clk_rlx(hdev); in goya_hw_fini()
2567 goya_set_pll_refclk(hdev); in goya_hw_fini()
2570 dev_info(hdev->dev, in goya_hw_fini()
2575 dev_info(hdev->dev, in goya_hw_fini()
2589 dev_err(hdev->dev, in goya_hw_fini()
2615 if (!hdev->pldm) { in goya_hw_fini()
2623 dev_info(hdev->dev, in goya_hw_fini()
2628 hdev, in goya_hw_fini()
2635 dev_err(hdev->dev, in goya_hw_fini()
2640 int goya_suspend(struct hl_device *hdev) in goya_suspend() argument
2644 rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS); in goya_suspend()
2646 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); in goya_suspend()
2651 int goya_resume(struct hl_device *hdev) in goya_resume() argument
2653 return goya_init_iatu(hdev); in goya_resume()
2656 static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma, in goya_cb_mmap() argument
2667 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in goya_cb_mmap()
2672 void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) in goya_ring_doorbell() argument
2739 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", in goya_ring_doorbell()
2754 void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) in goya_pqe_write() argument
2760 static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size, in goya_dma_alloc_coherent() argument
2763 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, in goya_dma_alloc_coherent()
2773 static void goya_dma_free_coherent(struct hl_device *hdev, size_t size, in goya_dma_free_coherent() argument
2779 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); in goya_dma_free_coherent()
2782 void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, in goya_get_int_queue_base() argument
2788 *dma_handle = hdev->asic_prop.sram_base_address; in goya_get_int_queue_base()
2790 base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; in goya_get_int_queue_base()
2830 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); in goya_get_int_queue_base()
2840 static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) in goya_send_job_on_qman0() argument
2849 if (hdev->pldm) in goya_send_job_on_qman0()
2854 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { in goya_send_job_on_qman0()
2855 dev_err_ratelimited(hdev->dev, in goya_send_job_on_qman0()
2860 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_send_job_on_qman0()
2863 dev_err(hdev->dev, in goya_send_job_on_qman0()
2868 goya_qman0_set_security(hdev, true); in goya_send_job_on_qman0()
2882 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0, in goya_send_job_on_qman0()
2885 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); in goya_send_job_on_qman0()
2889 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, in goya_send_job_on_qman0()
2893 hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0); in goya_send_job_on_qman0()
2896 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); in goya_send_job_on_qman0()
2901 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_send_job_on_qman0()
2904 goya_qman0_set_security(hdev, false); in goya_send_job_on_qman0()
2909 int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, in goya_send_cpu_message() argument
2912 struct goya_device *goya = hdev->asic_specific; in goya_send_cpu_message()
2920 return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len, in goya_send_cpu_message()
2924 int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id) in goya_test_queue() argument
2935 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_test_queue()
2938 dev_err(hdev->dev, in goya_test_queue()
2945 fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, in goya_test_queue()
2949 dev_err(hdev->dev, in goya_test_queue()
2962 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, in goya_test_queue()
2966 dev_err(hdev->dev, in goya_test_queue()
2971 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val), in goya_test_queue()
2974 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); in goya_test_queue()
2977 dev_err(hdev->dev, in goya_test_queue()
2982 dev_info(hdev->dev, "queue test on H/W queue %d succeeded\n", in goya_test_queue()
2987 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt, in goya_test_queue()
2990 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_test_queue()
2995 int goya_test_cpu_queue(struct hl_device *hdev) in goya_test_cpu_queue() argument
2997 struct goya_device *goya = hdev->asic_specific; in goya_test_cpu_queue()
3006 return hl_fw_test_cpu_queue(hdev); in goya_test_cpu_queue()
3009 int goya_test_queues(struct hl_device *hdev) in goya_test_queues() argument
3014 rc = goya_test_queue(hdev, i); in goya_test_queues()
3022 static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size, in goya_dma_pool_zalloc() argument
3030 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in goya_dma_pool_zalloc()
3039 static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr, in goya_dma_pool_free() argument
3045 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); in goya_dma_pool_free()
3048 void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, in goya_cpu_accessible_dma_pool_alloc() argument
3053 vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); in goya_cpu_accessible_dma_pool_alloc()
3054 *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + in goya_cpu_accessible_dma_pool_alloc()
3060 void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, in goya_cpu_accessible_dma_pool_free() argument
3063 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); in goya_cpu_accessible_dma_pool_free()
3066 static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl, in goya_dma_map_sg() argument
3072 if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir)) in goya_dma_map_sg()
3082 static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl, in goya_dma_unmap_sg() argument
3092 dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir); in goya_dma_unmap_sg()
3095 u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt) in goya_get_dma_desc_list_size() argument
3136 static int goya_pin_memory_before_cs(struct hl_device *hdev, in goya_pin_memory_before_cs() argument
3144 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3152 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3159 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl, in goya_pin_memory_before_cs()
3162 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in goya_pin_memory_before_cs()
3171 goya_get_dma_desc_list_size(hdev, userptr->sgt); in goya_pin_memory_before_cs()
3176 hl_unpin_host_memory(hdev, userptr); in goya_pin_memory_before_cs()
3182 static int goya_validate_dma_pkt_host(struct hl_device *hdev, in goya_validate_dma_pkt_host() argument
3205 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n"); in goya_validate_dma_pkt_host()
3215 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3223 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n"); in goya_validate_dma_pkt_host()
3232 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3238 dev_err(hdev->dev, "DMA direction is undefined\n"); in goya_validate_dma_pkt_host()
3245 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_host()
3246 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_host()
3248 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3257 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_host()
3258 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_host()
3260 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3273 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3278 rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt, in goya_validate_dma_pkt_host()
3285 static int goya_validate_dma_pkt_no_host(struct hl_device *hdev, in goya_validate_dma_pkt_no_host() argument
3298 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n"); in goya_validate_dma_pkt_no_host()
3302 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n"); in goya_validate_dma_pkt_no_host()
3309 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_no_host()
3310 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_no_host()
3311 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3318 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_no_host()
3319 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_no_host()
3320 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3330 static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev, in goya_validate_dma_pkt_no_mmu() argument
3338 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_no_mmu()
3339 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3341 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3343 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_no_mmu()
3354 dev_err(hdev->dev, in goya_validate_dma_pkt_no_mmu()
3360 rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt); in goya_validate_dma_pkt_no_mmu()
3362 rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt); in goya_validate_dma_pkt_no_mmu()
3367 static int goya_validate_dma_pkt_mmu(struct hl_device *hdev, in goya_validate_dma_pkt_mmu() argument
3371 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_mmu()
3372 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3374 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3376 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_mmu()
3385 hdev->asic_prop.va_space_host_start_address, in goya_validate_dma_pkt_mmu()
3386 hdev->asic_prop.va_space_host_end_address)) { in goya_validate_dma_pkt_mmu()
3387 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3393 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3403 static int goya_validate_wreg32(struct hl_device *hdev, in goya_validate_wreg32() argument
3407 struct goya_device *goya = hdev->asic_specific; in goya_validate_wreg32()
3414 dev_dbg(hdev->dev, "WREG32 packet details:\n"); in goya_validate_wreg32()
3415 dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset); in goya_validate_wreg32()
3416 dev_dbg(hdev->dev, "value == 0x%x\n", in goya_validate_wreg32()
3420 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n", in goya_validate_wreg32()
3439 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n", in goya_validate_wreg32()
3447 static int goya_validate_cb(struct hl_device *hdev, in goya_validate_cb() argument
3472 dev_err(hdev->dev, in goya_validate_cb()
3485 rc = goya_validate_wreg32(hdev, in goya_validate_cb()
3490 dev_err(hdev->dev, in goya_validate_cb()
3496 dev_err(hdev->dev, in goya_validate_cb()
3502 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_validate_cb()
3507 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_validate_cb()
3513 rc = goya_validate_dma_pkt_mmu(hdev, parser, in goya_validate_cb()
3516 rc = goya_validate_dma_pkt_no_mmu(hdev, parser, in goya_validate_cb()
3528 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_validate_cb()
3548 static int goya_patch_dma_packet(struct hl_device *hdev, in goya_patch_dma_packet() argument
3595 (hl_userptr_is_pinned(hdev, addr, in goya_patch_dma_packet()
3598 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", in goya_patch_dma_packet()
3664 dev_err(hdev->dev, in goya_patch_dma_packet()
3678 static int goya_patch_cb(struct hl_device *hdev, in goya_patch_cb() argument
3706 dev_err(hdev->dev, in goya_patch_cb()
3714 rc = goya_patch_dma_packet(hdev, parser, in goya_patch_cb()
3724 rc = goya_validate_wreg32(hdev, parser, in goya_patch_cb()
3729 dev_err(hdev->dev, in goya_patch_cb()
3735 dev_err(hdev->dev, in goya_patch_cb()
3741 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_patch_cb()
3746 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_patch_cb()
3759 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_patch_cb()
3772 static int goya_parse_cb_mmu(struct hl_device *hdev, in goya_parse_cb_mmu() argument
3788 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3793 dev_err(hdev->dev, in goya_parse_cb_mmu()
3800 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3823 rc = goya_validate_cb(hdev, parser, true); in goya_parse_cb_mmu()
3832 dev_err(hdev->dev, "user CB size mismatch\n"); in goya_parse_cb_mmu()
3845 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3851 static int goya_parse_cb_no_mmu(struct hl_device *hdev, in goya_parse_cb_no_mmu() argument
3857 rc = goya_validate_cb(hdev, parser, false); in goya_parse_cb_no_mmu()
3862 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
3866 dev_err(hdev->dev, in goya_parse_cb_no_mmu()
3872 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
3882 rc = goya_patch_cb(hdev, parser); in goya_parse_cb_no_mmu()
3894 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
3899 hl_userptr_delete_list(hdev, parser->job_userptr_list); in goya_parse_cb_no_mmu()
3903 static int goya_parse_cb_no_ext_queue(struct hl_device *hdev, in goya_parse_cb_no_ext_queue() argument
3906 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in goya_parse_cb_no_ext_queue()
3907 struct goya_device *goya = hdev->asic_specific; in goya_parse_cb_no_ext_queue()
3927 dev_err(hdev->dev, in goya_parse_cb_no_ext_queue()
3934 int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) in goya_cs_parser() argument
3936 struct goya_device *goya = hdev->asic_specific; in goya_cs_parser()
3939 return goya_parse_cb_no_ext_queue(hdev, parser); in goya_cs_parser()
3942 return goya_parse_cb_mmu(hdev, parser); in goya_cs_parser()
3944 return goya_parse_cb_no_mmu(hdev, parser); in goya_cs_parser()
3947 void goya_add_end_of_cb_packets(struct hl_device *hdev, u64 kernel_address, in goya_add_end_of_cb_packets() argument
3972 void goya_update_eq_ci(struct hl_device *hdev, u32 val) in goya_update_eq_ci() argument
3977 void goya_restore_phase_topology(struct hl_device *hdev) in goya_restore_phase_topology() argument
3982 static void goya_clear_sm_regs(struct hl_device *hdev) in goya_clear_sm_regs() argument
4017 static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) in goya_debugfs_read32() argument
4019 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_read32()
4029 *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_read32()
4033 (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size)) { in goya_debugfs_read32()
4038 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr); in goya_debugfs_read32()
4040 *val = readl(hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_read32()
4043 ddr_bar_addr = goya_set_ddr_bar_base(hdev, in goya_debugfs_read32()
4074 static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) in goya_debugfs_write32() argument
4076 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_write32()
4086 writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_write32()
4090 (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size)) { in goya_debugfs_write32()
4095 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr); in goya_debugfs_write32()
4097 writel(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_write32()
4100 ddr_bar_addr = goya_set_ddr_bar_base(hdev, in goya_debugfs_write32()
4116 static u64 goya_read_pte(struct hl_device *hdev, u64 addr) in goya_read_pte() argument
4118 struct goya_device *goya = hdev->asic_specific; in goya_read_pte()
4120 if (hdev->hard_reset_pending) in goya_read_pte()
4123 return readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_read_pte()
4127 static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val) in goya_write_pte() argument
4129 struct goya_device *goya = hdev->asic_specific; in goya_write_pte()
4131 if (hdev->hard_reset_pending) in goya_write_pte()
4134 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_write_pte()
4323 static void goya_print_razwi_info(struct hl_device *hdev) in goya_print_razwi_info() argument
4326 dev_err(hdev->dev, "Illegal write to LBW\n"); in goya_print_razwi_info()
4331 dev_err(hdev->dev, "Illegal read from LBW\n"); in goya_print_razwi_info()
4336 dev_err(hdev->dev, "Illegal write to HBW\n"); in goya_print_razwi_info()
4341 dev_err(hdev->dev, "Illegal read from HBW\n"); in goya_print_razwi_info()
4346 static void goya_print_mmu_error_info(struct hl_device *hdev) in goya_print_mmu_error_info() argument
4348 struct goya_device *goya = hdev->asic_specific; in goya_print_mmu_error_info()
4361 dev_err(hdev->dev, "MMU page fault on va 0x%llx\n", addr); in goya_print_mmu_error_info()
4367 static void goya_print_irq_info(struct hl_device *hdev, u16 event_type, in goya_print_irq_info() argument
4373 dev_err(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in goya_print_irq_info()
4377 goya_print_razwi_info(hdev); in goya_print_irq_info()
4378 goya_print_mmu_error_info(hdev); in goya_print_irq_info()
4382 static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr, in goya_unmask_irq_arr() argument
4400 dev_err(hdev->dev, "too many elements in IRQ array\n"); in goya_unmask_irq_arr()
4422 rc = goya_send_cpu_message(hdev, (u32 *) pkt, total_pkt_size, in goya_unmask_irq_arr()
4426 dev_err(hdev->dev, "failed to unmask IRQ array\n"); in goya_unmask_irq_arr()
4433 static int goya_soft_reset_late_init(struct hl_device *hdev) in goya_soft_reset_late_init() argument
4439 return goya_unmask_irq_arr(hdev, goya_all_events, in goya_soft_reset_late_init()
4443 static int goya_unmask_irq(struct hl_device *hdev, u16 event_type) in goya_unmask_irq() argument
4455 rc = goya_send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), in goya_unmask_irq()
4459 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type); in goya_unmask_irq()
4464 void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) in goya_handle_eqe() argument
4469 struct goya_device *goya = hdev->asic_specific; in goya_handle_eqe()
4498 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4499 hl_device_reset(hdev, true, false); in goya_handle_eqe()
4529 goya_print_irq_info(hdev, event_type, true); in goya_handle_eqe()
4530 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4543 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4544 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4548 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", in goya_handle_eqe()
4554 void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) in goya_get_events_stat() argument
4556 struct goya_device *goya = hdev->asic_specific; in goya_get_events_stat()
4567 static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, in goya_memset_device_memory() argument
4579 cb = hl_cb_kernel_create(hdev, cb_size); in goya_memset_device_memory()
4609 job = hl_cs_allocate_job(hdev, true); in goya_memset_device_memory()
4611 dev_err(hdev->dev, "Failed to allocate a new job\n"); in goya_memset_device_memory()
4624 hl_debugfs_add_job(hdev, job); in goya_memset_device_memory()
4626 rc = goya_send_job_on_qman0(hdev, job); in goya_memset_device_memory()
4630 hl_debugfs_remove_job(hdev, job); in goya_memset_device_memory()
4636 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); in goya_memset_device_memory()
4641 int goya_context_switch(struct hl_device *hdev, u32 asid) in goya_context_switch() argument
4643 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_context_switch()
4645 u32 size = hdev->pldm ? 0x10000 : prop->sram_size; in goya_context_switch()
4651 rc = goya_memset_device_memory(hdev, addr, size, val, false); in goya_context_switch()
4653 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n"); in goya_context_switch()
4670 goya_mmu_prepare(hdev, asid); in goya_context_switch()
4672 goya_clear_sm_regs(hdev); in goya_context_switch()
4677 static int goya_mmu_clear_pgt_range(struct hl_device *hdev) in goya_mmu_clear_pgt_range() argument
4679 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_clear_pgt_range()
4680 struct goya_device *goya = hdev->asic_specific; in goya_mmu_clear_pgt_range()
4688 return goya_memset_device_memory(hdev, addr, size, 0, true); in goya_mmu_clear_pgt_range()
4691 static int goya_mmu_set_dram_default_page(struct hl_device *hdev) in goya_mmu_set_dram_default_page() argument
4693 struct goya_device *goya = hdev->asic_specific; in goya_mmu_set_dram_default_page()
4694 u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; in goya_mmu_set_dram_default_page()
4701 return goya_memset_device_memory(hdev, addr, size, val, true); in goya_mmu_set_dram_default_page()
4704 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev) in goya_mmu_add_mappings_for_device_cpu() argument
4706 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_add_mappings_for_device_cpu()
4707 struct goya_device *goya = hdev->asic_specific; in goya_mmu_add_mappings_for_device_cpu()
4715 rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off, in goya_mmu_add_mappings_for_device_cpu()
4718 dev_err(hdev->dev, "Map failed for address 0x%llx\n", in goya_mmu_add_mappings_for_device_cpu()
4724 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_add_mappings_for_device_cpu()
4725 rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, in goya_mmu_add_mappings_for_device_cpu()
4726 hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB); in goya_mmu_add_mappings_for_device_cpu()
4729 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4736 rc = hl_mmu_map(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4738 hdev->cpu_accessible_dma_address + cpu_off, in goya_mmu_add_mappings_for_device_cpu()
4741 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4749 goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID); in goya_mmu_add_mappings_for_device_cpu()
4750 goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID); in goya_mmu_add_mappings_for_device_cpu()
4763 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4766 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4771 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4773 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4780 void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev) in goya_mmu_remove_device_cpu_mappings() argument
4782 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_remove_device_cpu_mappings()
4783 struct goya_device *goya = hdev->asic_specific; in goya_mmu_remove_device_cpu_mappings()
4795 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_remove_device_cpu_mappings()
4796 if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, in goya_mmu_remove_device_cpu_mappings()
4798 dev_warn(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4802 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4805 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4811 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4813 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4820 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) in goya_mmu_prepare() argument
4822 struct goya_device *goya = hdev->asic_specific; in goya_mmu_prepare()
4835 goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid); in goya_mmu_prepare()
4838 static void goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard) in goya_mmu_invalidate_cache() argument
4840 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache()
4851 if (hdev->pldm) in goya_mmu_invalidate_cache()
4856 mutex_lock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache()
4862 hdev, in goya_mmu_invalidate_cache()
4869 mutex_unlock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache()
4872 dev_notice_ratelimited(hdev->dev, in goya_mmu_invalidate_cache()
4876 static void goya_mmu_invalidate_cache_range(struct hl_device *hdev, in goya_mmu_invalidate_cache_range() argument
4879 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache_range()
4890 if (hdev->pldm) in goya_mmu_invalidate_cache_range()
4895 mutex_lock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache_range()
4912 hdev, in goya_mmu_invalidate_cache_range()
4919 mutex_unlock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache_range()
4922 dev_notice_ratelimited(hdev->dev, in goya_mmu_invalidate_cache_range()
4926 int goya_send_heartbeat(struct hl_device *hdev) in goya_send_heartbeat() argument
4928 struct goya_device *goya = hdev->asic_specific; in goya_send_heartbeat()
4933 return hl_fw_send_heartbeat(hdev); in goya_send_heartbeat()
4936 int goya_armcp_info_get(struct hl_device *hdev) in goya_armcp_info_get() argument
4938 struct goya_device *goya = hdev->asic_specific; in goya_armcp_info_get()
4939 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_armcp_info_get()
4946 rc = hl_fw_armcp_info_get(hdev); in goya_armcp_info_get()
4954 dev_err(hdev->dev, in goya_armcp_info_get()
4971 static bool goya_is_device_idle(struct hl_device *hdev, u32 *mask, in goya_is_device_idle() argument
5049 static void goya_hw_queues_lock(struct hl_device *hdev) in goya_hw_queues_lock() argument
5051 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_lock()
5056 static void goya_hw_queues_unlock(struct hl_device *hdev) in goya_hw_queues_unlock() argument
5058 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_unlock()
5063 static u32 goya_get_pci_id(struct hl_device *hdev) in goya_get_pci_id() argument
5065 return hdev->pdev->device; in goya_get_pci_id()
5068 static int goya_get_eeprom_data(struct hl_device *hdev, void *data, in goya_get_eeprom_data() argument
5071 struct goya_device *goya = hdev->asic_specific; in goya_get_eeprom_data()
5076 return hl_fw_get_eeprom_data(hdev, data, max_size); in goya_get_eeprom_data()
5079 static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev) in goya_get_hw_state() argument
5149 void goya_set_asic_funcs(struct hl_device *hdev) in goya_set_asic_funcs() argument
5151 hdev->asic_funcs = &goya_funcs; in goya_set_asic_funcs()