Lines Matching +full:mmu +full:- +full:500
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
9 #include "../include/hw_ip/mmu/mmu_general.h"
10 #include "../include/hw_ip/mmu/mmu_v1_0.h"
17 #include <linux/io-64-nonatomic-lo-hi.h>
25 * - Range registers (When MMU is enabled, DMA RR does NOT protect host)
26 * - MMU
29 * - Range registers (protect the first 512MB)
30 * - MMU (isolation between users)
33 * - Range registers
34 * - Protection bits
36 * When MMU is disabled:
46 * - checks DMA pointer
47 * - WREG, MSG_PROT are not allowed.
48 * - MSG_LONG/SHORT are allowed.
54 * When MMU is enabled:
57 * MMU is set to bypass on the Secure props register of the QMAN.
58 * The reasons we don't enable MMU for PQ, CQ and CP are:
59 * - PQ entry is in kernel address space and the driver doesn't map it.
60 * - CP writes to MSIX register and to kernel address space (completion
67 * the driver doesn't map memory in MMU.
69 * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode)
75 #define GOYA_BOOT_FIT_FILE "habanalabs/goya/goya-boot-fit.itb"
76 #define GOYA_LINUX_FW_FILE "habanalabs/goya/goya-fit.itb"
82 #define GOYA_RESET_TIMEOUT_MSEC 500 /* 500ms */
362 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_get_fixed_properties()
365 prop->max_queues = GOYA_QUEUE_ID_SIZE; in goya_get_fixed_properties()
366 prop->hw_queues_props = kcalloc(prop->max_queues, in goya_get_fixed_properties()
370 if (!prop->hw_queues_props) in goya_get_fixed_properties()
371 return -ENOMEM; in goya_get_fixed_properties()
374 prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; in goya_get_fixed_properties()
375 prop->hw_queues_props[i].driver_only = 0; in goya_get_fixed_properties()
376 prop->hw_queues_props[i].requires_kernel_cb = 1; in goya_get_fixed_properties()
380 prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; in goya_get_fixed_properties()
381 prop->hw_queues_props[i].driver_only = 1; in goya_get_fixed_properties()
382 prop->hw_queues_props[i].requires_kernel_cb = 0; in goya_get_fixed_properties()
387 prop->hw_queues_props[i].type = QUEUE_TYPE_INT; in goya_get_fixed_properties()
388 prop->hw_queues_props[i].driver_only = 0; in goya_get_fixed_properties()
389 prop->hw_queues_props[i].requires_kernel_cb = 0; in goya_get_fixed_properties()
392 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; in goya_get_fixed_properties()
394 prop->dram_base_address = DRAM_PHYS_BASE; in goya_get_fixed_properties()
395 prop->dram_size = DRAM_PHYS_DEFAULT_SIZE; in goya_get_fixed_properties()
396 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in goya_get_fixed_properties()
397 prop->dram_user_base_address = DRAM_BASE_ADDR_USER; in goya_get_fixed_properties()
399 prop->sram_base_address = SRAM_BASE_ADDR; in goya_get_fixed_properties()
400 prop->sram_size = SRAM_SIZE; in goya_get_fixed_properties()
401 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in goya_get_fixed_properties()
402 prop->sram_user_base_address = prop->sram_base_address + in goya_get_fixed_properties()
405 prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR; in goya_get_fixed_properties()
406 prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR; in goya_get_fixed_properties()
407 if (hdev->pldm) in goya_get_fixed_properties()
408 prop->mmu_pgt_size = 0x800000; /* 8MB */ in goya_get_fixed_properties()
410 prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE; in goya_get_fixed_properties()
411 prop->mmu_pte_size = HL_PTE_SIZE; in goya_get_fixed_properties()
412 prop->mmu_hop_table_size = HOP_TABLE_SIZE; in goya_get_fixed_properties()
413 prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE; in goya_get_fixed_properties()
414 prop->dram_page_size = PAGE_SIZE_2MB; in goya_get_fixed_properties()
416 prop->dmmu.hop0_shift = HOP0_SHIFT; in goya_get_fixed_properties()
417 prop->dmmu.hop1_shift = HOP1_SHIFT; in goya_get_fixed_properties()
418 prop->dmmu.hop2_shift = HOP2_SHIFT; in goya_get_fixed_properties()
419 prop->dmmu.hop3_shift = HOP3_SHIFT; in goya_get_fixed_properties()
420 prop->dmmu.hop4_shift = HOP4_SHIFT; in goya_get_fixed_properties()
421 prop->dmmu.hop0_mask = HOP0_MASK; in goya_get_fixed_properties()
422 prop->dmmu.hop1_mask = HOP1_MASK; in goya_get_fixed_properties()
423 prop->dmmu.hop2_mask = HOP2_MASK; in goya_get_fixed_properties()
424 prop->dmmu.hop3_mask = HOP3_MASK; in goya_get_fixed_properties()
425 prop->dmmu.hop4_mask = HOP4_MASK; in goya_get_fixed_properties()
426 prop->dmmu.start_addr = VA_DDR_SPACE_START; in goya_get_fixed_properties()
427 prop->dmmu.end_addr = VA_DDR_SPACE_END; in goya_get_fixed_properties()
428 prop->dmmu.page_size = PAGE_SIZE_2MB; in goya_get_fixed_properties()
429 prop->dmmu.num_hops = MMU_ARCH_5_HOPS; in goya_get_fixed_properties()
432 memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu)); in goya_get_fixed_properties()
433 prop->pmmu.start_addr = VA_HOST_SPACE_START; in goya_get_fixed_properties()
434 prop->pmmu.end_addr = VA_HOST_SPACE_END; in goya_get_fixed_properties()
435 prop->pmmu.page_size = PAGE_SIZE_4KB; in goya_get_fixed_properties()
436 prop->pmmu.num_hops = MMU_ARCH_5_HOPS; in goya_get_fixed_properties()
439 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in goya_get_fixed_properties()
440 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in goya_get_fixed_properties()
442 prop->dram_size_for_default_page_mapping = VA_DDR_SPACE_END; in goya_get_fixed_properties()
443 prop->cfg_size = CFG_SIZE; in goya_get_fixed_properties()
444 prop->max_asid = MAX_ASID; in goya_get_fixed_properties()
445 prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE; in goya_get_fixed_properties()
446 prop->high_pll = PLL_HIGH_DEFAULT; in goya_get_fixed_properties()
447 prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT; in goya_get_fixed_properties()
448 prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE; in goya_get_fixed_properties()
449 prop->max_power_default = MAX_POWER_DEFAULT; in goya_get_fixed_properties()
450 prop->tpc_enabled_mask = TPC_ENABLED_MASK; in goya_get_fixed_properties()
451 prop->pcie_dbi_base_address = mmPCIE_DBI_BASE; in goya_get_fixed_properties()
452 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in goya_get_fixed_properties()
454 strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, in goya_get_fixed_properties()
457 prop->max_pending_cs = GOYA_MAX_PENDING_CS; in goya_get_fixed_properties()
463 * goya_pci_bars_map - Map PCI BARS of Goya device
481 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_pci_bars_map()
482 (CFG_BASE - SRAM_BASE_ADDR); in goya_pci_bars_map()
489 struct goya_device *goya = hdev->asic_specific; in goya_set_ddr_bar_base()
494 if ((goya) && (goya->ddr_bar_cur_addr == addr)) in goya_set_ddr_bar_base()
497 /* Inbound Region 1 - Bar 4 - Point to DDR */ in goya_set_ddr_bar_base()
506 old_addr = goya->ddr_bar_cur_addr; in goya_set_ddr_bar_base()
507 goya->ddr_bar_cur_addr = addr; in goya_set_ddr_bar_base()
514 * goya_init_iatu - Initialize the iATU unit inside the PCI controller
527 /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */ in goya_init_iatu()
535 /* Inbound Region 1 - Bar 4 - Point to DDR */ in goya_init_iatu()
543 hdev->asic_funcs->set_dma_mask_from_fw(hdev); in goya_init_iatu()
545 /* Outbound Region 0 - Point to Host */ in goya_init_iatu()
555 * goya_early_init - GOYA early initialization code
567 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_early_init()
568 struct pci_dev *pdev = hdev->pdev; in goya_early_init()
574 dev_err(hdev->dev, "Failed to get fixed properties\n"); in goya_early_init()
580 dev_err(hdev->dev, in goya_early_init()
586 rc = -ENODEV; in goya_early_init()
591 dev_err(hdev->dev, in goya_early_init()
597 rc = -ENODEV; in goya_early_init()
601 prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID); in goya_early_init()
609 prop->fw_security_disabled = true; in goya_early_init()
610 dev_info(hdev->dev, "firmware-level security is disabled\n"); in goya_early_init()
612 if (!hdev->pldm) { in goya_early_init()
615 dev_warn(hdev->dev, in goya_early_init()
622 kfree(hdev->asic_prop.hw_queues_props); in goya_early_init()
627 * goya_early_fini - GOYA early finalization code
636 kfree(hdev->asic_prop.hw_queues_props); in goya_early_fini()
651 struct goya_device *goya = hdev->asic_specific; in goya_qman0_set_security()
653 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_qman0_set_security()
665 * goya_fetch_psoc_frequency - Fetch PSOC frequency values
672 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_fetch_psoc_frequency()
694 dev_warn(hdev->dev, in goya_fetch_psoc_frequency()
698 prop->psoc_timestamp_frequency = trace_freq; in goya_fetch_psoc_frequency()
699 prop->psoc_pci_pll_nr = nr; in goya_fetch_psoc_frequency()
700 prop->psoc_pci_pll_nf = nf; in goya_fetch_psoc_frequency()
701 prop->psoc_pci_pll_od = od; in goya_fetch_psoc_frequency()
702 prop->psoc_pci_pll_div_factor = div_fctr; in goya_fetch_psoc_frequency()
707 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_late_init()
714 dev_err(hdev->dev, in goya_late_init()
715 "Failed to clear MMU page tables range %d\n", rc); in goya_late_init()
721 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); in goya_late_init()
739 dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc); in goya_late_init()
745 * the MMU block) accordingly. The value is the log2 of the DRAM size in goya_late_init()
747 WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size)); in goya_late_init()
751 dev_err(hdev->dev, in goya_late_init()
763 * goya_late_fini - GOYA late tear-down code
774 if (!hdev->hl_chip_info->info) in goya_late_fini()
777 channel_info_arr = hdev->hl_chip_info->info; in goya_late_fini()
780 kfree(channel_info_arr[i]->config); in goya_late_fini()
787 hdev->hl_chip_info->info = NULL; in goya_late_fini()
791 * goya_sw_init - Goya software initialization code
804 return -ENOMEM; in goya_sw_init()
807 goya->ddr_bar_cur_addr = DRAM_PHYS_BASE; in goya_sw_init()
809 goya->mme_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
810 goya->tpc_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
811 goya->ic_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
813 hdev->asic_specific = goya; in goya_sw_init()
816 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), in goya_sw_init()
817 &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0); in goya_sw_init()
818 if (!hdev->dma_pool) { in goya_sw_init()
819 dev_err(hdev->dev, "failed to create DMA pool\n"); in goya_sw_init()
820 rc = -ENOMEM; in goya_sw_init()
824 hdev->cpu_accessible_dma_mem = in goya_sw_init()
825 hdev->asic_funcs->asic_dma_alloc_coherent(hdev, in goya_sw_init()
827 &hdev->cpu_accessible_dma_address, in goya_sw_init()
830 if (!hdev->cpu_accessible_dma_mem) { in goya_sw_init()
831 rc = -ENOMEM; in goya_sw_init()
835 dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n", in goya_sw_init()
836 &hdev->cpu_accessible_dma_address); in goya_sw_init()
838 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in goya_sw_init()
839 if (!hdev->cpu_accessible_dma_pool) { in goya_sw_init()
840 dev_err(hdev->dev, in goya_sw_init()
842 rc = -ENOMEM; in goya_sw_init()
846 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, in goya_sw_init()
847 (uintptr_t) hdev->cpu_accessible_dma_mem, in goya_sw_init()
848 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in goya_sw_init()
850 dev_err(hdev->dev, in goya_sw_init()
852 rc = -EFAULT; in goya_sw_init()
856 spin_lock_init(&goya->hw_queues_lock); in goya_sw_init()
857 hdev->supports_coresight = true; in goya_sw_init()
858 hdev->supports_soft_reset = true; in goya_sw_init()
863 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_init()
865 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_init()
867 hdev->cpu_accessible_dma_mem, in goya_sw_init()
868 hdev->cpu_accessible_dma_address); in goya_sw_init()
870 dma_pool_destroy(hdev->dma_pool); in goya_sw_init()
878 * goya_sw_fini - Goya software tear-down code
885 struct goya_device *goya = hdev->asic_specific; in goya_sw_fini()
887 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_fini()
889 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_fini()
891 hdev->cpu_accessible_dma_mem, in goya_sw_fini()
892 hdev->cpu_accessible_dma_address); in goya_sw_fini()
894 dma_pool_destroy(hdev->dma_pool); in goya_sw_fini()
904 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qman()
908 u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI); in goya_init_dma_qman()
941 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_init_dma_qman()
946 if (hdev->stop_on_err) in goya_init_dma_qman()
957 u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1); in goya_init_dma_ch()
971 (dma_id - 1) * 4; in goya_init_dma_ch()
980 * goya_init_dma_qmans - Initialize QMAN DMA registers
989 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qmans()
993 if (goya->hw_cap_initialized & HW_CAP_DMA) in goya_init_dma_qmans()
996 q = &hdev->kernel_queues[0]; in goya_init_dma_qmans()
999 q->cq_id = q->msi_vec = i; in goya_init_dma_qmans()
1000 goya_init_dma_qman(hdev, i, q->bus_address); in goya_init_dma_qmans()
1004 goya->hw_cap_initialized |= HW_CAP_DMA; in goya_init_dma_qmans()
1008 * goya_disable_external_queues - Disable external queues
1015 struct goya_device *goya = hdev->asic_specific; in goya_disable_external_queues()
1017 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_disable_external_queues()
1061 dev_err(hdev->dev, in goya_stop_queue()
1063 return -EINVAL; in goya_stop_queue()
1070 * goya_stop_external_queues - Stop external queues
1081 struct goya_device *goya = hdev->asic_specific; in goya_stop_external_queues()
1083 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_stop_external_queues()
1092 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n"); in goya_stop_external_queues()
1093 retval = -EIO; in goya_stop_external_queues()
1102 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n"); in goya_stop_external_queues()
1103 retval = -EIO; in goya_stop_external_queues()
1112 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n"); in goya_stop_external_queues()
1113 retval = -EIO; in goya_stop_external_queues()
1122 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n"); in goya_stop_external_queues()
1123 retval = -EIO; in goya_stop_external_queues()
1132 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n"); in goya_stop_external_queues()
1133 retval = -EIO; in goya_stop_external_queues()
1140 * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
1149 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu_queues()
1152 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_init_cpu_queues()
1155 if (!hdev->cpu_queues_enable) in goya_init_cpu_queues()
1158 if (goya->hw_cap_initialized & HW_CAP_CPU_Q) in goya_init_cpu_queues()
1161 eq = &hdev->event_queue; in goya_init_cpu_queues()
1163 WREG32(mmCPU_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in goya_init_cpu_queues()
1164 WREG32(mmCPU_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in goya_init_cpu_queues()
1166 WREG32(mmCPU_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in goya_init_cpu_queues()
1167 WREG32(mmCPU_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in goya_init_cpu_queues()
1197 dev_err(hdev->dev, in goya_init_cpu_queues()
1199 return -EIO; in goya_init_cpu_queues()
1202 goya->hw_cap_initialized |= HW_CAP_CPU_Q; in goya_init_cpu_queues()
1258 tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE); in _goya_tpc_mbist_workaround()
1268 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n", in _goya_tpc_mbist_workaround()
1296 dev_err(hdev->dev, in _goya_tpc_mbist_workaround()
1317 struct goya_device *goya = hdev->asic_specific; in goya_tpc_mbist_workaround()
1320 if (hdev->pldm) in goya_tpc_mbist_workaround()
1323 if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST) in goya_tpc_mbist_workaround()
1331 goya->hw_cap_initialized |= HW_CAP_TPC_MBIST; in goya_tpc_mbist_workaround()
1335 * goya_init_golden_registers - Initialize golden registers
1344 struct goya_device *goya = hdev->asic_specific; in goya_init_golden_registers()
1348 if (goya->hw_cap_initialized & HW_CAP_GOLDEN) in goya_init_golden_registers()
1624 * Workaround for H2 #HW-23 bug in goya_init_golden_registers()
1634 goya->hw_cap_initialized |= HW_CAP_GOLDEN; in goya_init_golden_registers()
1654 qman_base_addr = hdev->asic_prop.sram_base_address + in goya_init_mme_qman()
1725 struct goya_device *goya = hdev->asic_specific; in goya_init_mme_qmans()
1728 if (goya->hw_cap_initialized & HW_CAP_MME) in goya_init_mme_qmans()
1740 goya->hw_cap_initialized |= HW_CAP_MME; in goya_init_mme_qmans()
1749 u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI); in goya_init_tpc_qman()
1761 qman_base_addr = hdev->asic_prop.sram_base_address + base_off; in goya_init_tpc_qman()
1798 u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1); in goya_init_tpc_cmdq()
1832 struct goya_device *goya = hdev->asic_specific; in goya_init_tpc_qmans()
1834 u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW - in goya_init_tpc_qmans()
1838 if (goya->hw_cap_initialized & HW_CAP_TPC) in goya_init_tpc_qmans()
1863 goya->hw_cap_initialized |= HW_CAP_TPC; in goya_init_tpc_qmans()
1867 * goya_disable_internal_queues - Disable internal queues
1874 struct goya_device *goya = hdev->asic_specific; in goya_disable_internal_queues()
1876 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_disable_internal_queues()
1883 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_disable_internal_queues()
1912 * goya_stop_internal_queues - Stop internal queues
1921 struct goya_device *goya = hdev->asic_specific; in goya_stop_internal_queues()
1924 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_stop_internal_queues()
1939 dev_err(hdev->dev, "failed to stop MME QMAN\n"); in goya_stop_internal_queues()
1940 retval = -EIO; in goya_stop_internal_queues()
1949 dev_err(hdev->dev, "failed to stop MME CMDQ\n"); in goya_stop_internal_queues()
1950 retval = -EIO; in goya_stop_internal_queues()
1954 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_stop_internal_queues()
1963 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n"); in goya_stop_internal_queues()
1964 retval = -EIO; in goya_stop_internal_queues()
1973 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n"); in goya_stop_internal_queues()
1974 retval = -EIO; in goya_stop_internal_queues()
1983 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n"); in goya_stop_internal_queues()
1984 retval = -EIO; in goya_stop_internal_queues()
1993 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n"); in goya_stop_internal_queues()
1994 retval = -EIO; in goya_stop_internal_queues()
2003 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n"); in goya_stop_internal_queues()
2004 retval = -EIO; in goya_stop_internal_queues()
2013 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n"); in goya_stop_internal_queues()
2014 retval = -EIO; in goya_stop_internal_queues()
2023 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n"); in goya_stop_internal_queues()
2024 retval = -EIO; in goya_stop_internal_queues()
2033 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n"); in goya_stop_internal_queues()
2034 retval = -EIO; in goya_stop_internal_queues()
2043 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n"); in goya_stop_internal_queues()
2044 retval = -EIO; in goya_stop_internal_queues()
2053 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n"); in goya_stop_internal_queues()
2054 retval = -EIO; in goya_stop_internal_queues()
2063 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n"); in goya_stop_internal_queues()
2064 retval = -EIO; in goya_stop_internal_queues()
2073 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n"); in goya_stop_internal_queues()
2074 retval = -EIO; in goya_stop_internal_queues()
2083 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n"); in goya_stop_internal_queues()
2084 retval = -EIO; in goya_stop_internal_queues()
2093 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n"); in goya_stop_internal_queues()
2094 retval = -EIO; in goya_stop_internal_queues()
2103 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n"); in goya_stop_internal_queues()
2104 retval = -EIO; in goya_stop_internal_queues()
2113 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n"); in goya_stop_internal_queues()
2114 retval = -EIO; in goya_stop_internal_queues()
2122 struct goya_device *goya = hdev->asic_specific; in goya_dma_stall()
2124 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_dma_stall()
2136 struct goya_device *goya = hdev->asic_specific; in goya_tpc_stall()
2138 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_tpc_stall()
2153 struct goya_device *goya = hdev->asic_specific; in goya_mme_stall()
2155 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_mme_stall()
2163 struct goya_device *goya = hdev->asic_specific; in goya_enable_msix()
2164 int cq_cnt = hdev->asic_prop.completion_queues_count; in goya_enable_msix()
2167 if (goya->hw_cap_initialized & HW_CAP_MSIX) in goya_enable_msix()
2170 rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES, in goya_enable_msix()
2173 dev_err(hdev->dev, in goya_enable_msix()
2174 "MSI-X: Failed to enable support -- %d/%d\n", in goya_enable_msix()
2180 irq = pci_irq_vector(hdev->pdev, i); in goya_enable_msix()
2182 &hdev->completion_queue[i]); in goya_enable_msix()
2184 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2189 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_enable_msix()
2193 &hdev->event_queue); in goya_enable_msix()
2195 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2199 goya->hw_cap_initialized |= HW_CAP_MSIX; in goya_enable_msix()
2204 free_irq(pci_irq_vector(hdev->pdev, i), in goya_enable_msix()
2205 &hdev->completion_queue[i]); in goya_enable_msix()
2207 pci_free_irq_vectors(hdev->pdev); in goya_enable_msix()
2213 struct goya_device *goya = hdev->asic_specific; in goya_sync_irqs()
2216 if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) in goya_sync_irqs()
2220 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in goya_sync_irqs()
2221 synchronize_irq(pci_irq_vector(hdev->pdev, i)); in goya_sync_irqs()
2223 synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX)); in goya_sync_irqs()
2228 struct goya_device *goya = hdev->asic_specific; in goya_disable_msix()
2231 if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) in goya_disable_msix()
2236 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_disable_msix()
2237 free_irq(irq, &hdev->event_queue); in goya_disable_msix()
2239 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in goya_disable_msix()
2240 irq = pci_irq_vector(hdev->pdev, i); in goya_disable_msix()
2241 free_irq(irq, &hdev->completion_queue[i]); in goya_disable_msix()
2244 pci_free_irq_vectors(hdev->pdev); in goya_disable_msix()
2246 goya->hw_cap_initialized &= ~HW_CAP_MSIX; in goya_disable_msix()
2252 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in goya_enable_timestamp()
2254 /* Zero the lower/upper parts of the 64-bit counter */ in goya_enable_timestamp()
2255 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0); in goya_enable_timestamp()
2256 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0); in goya_enable_timestamp()
2259 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1); in goya_enable_timestamp()
2265 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in goya_disable_timestamp()
2272 dev_info(hdev->dev, in goya_halt_engines()
2275 if (hdev->pldm) in goya_halt_engines()
2305 * goya_load_firmware_to_device() - Load LINUX FW code to device.
2310 * Return: 0 on success, non-zero for failure.
2316 dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET; in goya_load_firmware_to_device()
2322 * goya_load_boot_fit_to_device() - Load boot fit to device.
2327 * Return: 0 on success, non-zero for failure.
2333 dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET; in goya_load_boot_fit_to_device()
2352 dest = hdev->asic_prop.uboot_ver; in goya_read_device_fw_version()
2353 name = "U-Boot"; in goya_read_device_fw_version()
2357 dest = hdev->asic_prop.preboot_ver; in goya_read_device_fw_version()
2361 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); in goya_read_device_fw_version()
2367 if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) { in goya_read_device_fw_version()
2368 memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off, in goya_read_device_fw_version()
2371 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", in goya_read_device_fw_version()
2379 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu()
2382 if (!hdev->cpu_enable) in goya_init_cpu()
2385 if (goya->hw_cap_initialized & HW_CAP_CPU) in goya_init_cpu()
2389 * Before pushing u-boot/linux to device, need to set the ddr bar to in goya_init_cpu()
2393 dev_err(hdev->dev, in goya_init_cpu()
2395 return -EIO; in goya_init_cpu()
2407 goya->hw_cap_initialized |= HW_CAP_CPU; in goya_init_cpu()
2418 if (hdev->pldm) in goya_mmu_update_asid_hop0_addr()
2436 dev_err(hdev->dev, in goya_mmu_update_asid_hop0_addr()
2437 "Timeout during MMU hop0 config of asid %d\n", asid); in goya_mmu_update_asid_hop0_addr()
2446 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_init()
2447 struct goya_device *goya = hdev->asic_specific; in goya_mmu_init()
2451 if (!hdev->mmu_enable) in goya_mmu_init()
2454 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_mmu_init()
2457 hdev->dram_supports_virtual_memory = true; in goya_mmu_init()
2458 hdev->dram_default_page_mapping = true; in goya_mmu_init()
2460 for (i = 0 ; i < prop->max_asid ; i++) { in goya_mmu_init()
2461 hop0_addr = prop->mmu_pgt_addr + in goya_mmu_init()
2462 (i * prop->mmu_hop_table_size); in goya_mmu_init()
2466 dev_err(hdev->dev, in goya_mmu_init()
2472 goya->hw_cap_initialized |= HW_CAP_MMU; in goya_mmu_init()
2474 /* init MMU cache manage page */ in goya_mmu_init()
2483 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, in goya_mmu_init()
2496 * goya_hw_init - Goya hardware initialization code
2505 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_hw_init()
2508 dev_info(hdev->dev, "Starting initialization of H/W\n"); in goya_hw_init()
2523 dev_err(hdev->dev, "failed to initialize CPU\n"); in goya_hw_init()
2533 * iATU to point to the start address of the MMU page tables in goya_hw_init()
2536 ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) { in goya_hw_init()
2537 dev_err(hdev->dev, in goya_hw_init()
2538 "failed to map DDR bar to MMU page tables\n"); in goya_hw_init()
2539 return -EIO; in goya_hw_init()
2556 /* MSI-X must be enabled before CPU queues are initialized */ in goya_hw_init()
2561 /* Perform read from the device to flush all MSI-X configuration */ in goya_hw_init()
2574 * goya_hw_fini - Goya hardware tear-down code
2582 struct goya_device *goya = hdev->asic_specific; in goya_hw_fini()
2585 if (hdev->pldm) { in goya_hw_fini()
2608 dev_info(hdev->dev, in goya_hw_fini()
2613 dev_info(hdev->dev, in goya_hw_fini()
2627 dev_err(hdev->dev, in goya_hw_fini()
2632 goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME | in goya_hw_fini()
2639 /* Chicken bit to re-initiate boot sequencer flow */ in goya_hw_fini()
2646 goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | in goya_hw_fini()
2651 memset(goya->events_stat, 0, sizeof(goya->events_stat)); in goya_hw_fini()
2660 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); in goya_suspend()
2675 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | in goya_cb_mmap()
2678 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); in goya_cb_mmap()
2680 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in goya_cb_mmap()
2752 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", in goya_ring_doorbell()
2776 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, in goya_dma_alloc_coherent()
2790 dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE; in goya_dma_free_coherent()
2792 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); in goya_dma_free_coherent()
2801 *dma_handle = hdev->asic_prop.sram_base_address; in goya_get_int_queue_base()
2803 base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; in goya_get_int_queue_base()
2843 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); in goya_get_int_queue_base()
2862 if (hdev->pldm) in goya_send_job_on_qman0()
2867 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { in goya_send_job_on_qman0()
2868 dev_err_ratelimited(hdev->dev, in goya_send_job_on_qman0()
2870 return -EBUSY; in goya_send_job_on_qman0()
2873 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_send_job_on_qman0()
2876 dev_err(hdev->dev, in goya_send_job_on_qman0()
2878 return -ENOMEM; in goya_send_job_on_qman0()
2883 cb = job->patched_cb; in goya_send_job_on_qman0()
2885 fence_pkt = cb->kernel_address + in goya_send_job_on_qman0()
2886 job->job_cb_size - sizeof(struct packet_msg_prot); in goya_send_job_on_qman0()
2891 fence_pkt->ctl = cpu_to_le32(tmp); in goya_send_job_on_qman0()
2892 fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL); in goya_send_job_on_qman0()
2893 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in goya_send_job_on_qman0()
2896 job->job_cb_size, cb->bus_address); in goya_send_job_on_qman0()
2898 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); in goya_send_job_on_qman0()
2908 if (rc == -ETIMEDOUT) { in goya_send_job_on_qman0()
2909 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); in goya_send_job_on_qman0()
2914 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_send_job_on_qman0()
2925 struct goya_device *goya = hdev->asic_specific; in goya_send_cpu_message()
2927 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) { in goya_send_cpu_message()
2951 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_test_queue()
2954 dev_err(hdev->dev, in goya_test_queue()
2957 return -ENOMEM; in goya_test_queue()
2962 fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, in goya_test_queue()
2966 dev_err(hdev->dev, in goya_test_queue()
2969 rc = -ENOMEM; in goya_test_queue()
2976 fence_pkt->ctl = cpu_to_le32(tmp); in goya_test_queue()
2977 fence_pkt->value = cpu_to_le32(fence_val); in goya_test_queue()
2978 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in goya_test_queue()
2984 dev_err(hdev->dev, in goya_test_queue()
2995 if (rc == -ETIMEDOUT) { in goya_test_queue()
2996 dev_err(hdev->dev, in goya_test_queue()
2999 rc = -EIO; in goya_test_queue()
3003 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt, in goya_test_queue()
3006 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_test_queue()
3013 struct goya_device *goya = hdev->asic_specific; in goya_test_cpu_queue()
3019 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_test_cpu_queue()
3032 ret_val = -EINVAL; in goya_test_queues()
3046 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in goya_dma_pool_zalloc()
3059 dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE; in goya_dma_pool_free()
3061 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); in goya_dma_pool_free()
3070 *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + in goya_cpu_accessible_dma_pool_alloc()
3088 if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir)) in goya_dma_map_sg()
3089 return -ENOMEM; in goya_dma_map_sg()
3093 sg->dma_address += HOST_PHYS_BASE; in goya_dma_map_sg()
3106 sg->dma_address -= HOST_PHYS_BASE; in goya_dma_unmap_sg()
3108 dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir); in goya_dma_unmap_sg()
3120 for_each_sg(sgt->sgl, sg, sgt->nents, count) { in goya_get_dma_desc_list_size()
3128 while ((count + 1) < sgt->nents) { in goya_get_dma_desc_list_size()
3160 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3161 parser->job_userptr_list, &userptr)) in goya_pin_memory_before_cs()
3166 return -ENOMEM; in goya_pin_memory_before_cs()
3168 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3173 list_add_tail(&userptr->job_node, parser->job_userptr_list); in goya_pin_memory_before_cs()
3175 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl, in goya_pin_memory_before_cs()
3176 userptr->sgt->nents, dir); in goya_pin_memory_before_cs()
3178 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in goya_pin_memory_before_cs()
3182 userptr->dma_mapped = true; in goya_pin_memory_before_cs()
3183 userptr->dir = dir; in goya_pin_memory_before_cs()
3186 parser->patched_cb_size += in goya_pin_memory_before_cs()
3187 goya_get_dma_desc_list_size(hdev, userptr->sgt); in goya_pin_memory_before_cs()
3211 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_host()
3221 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n"); in goya_validate_dma_pkt_host()
3224 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3225 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3231 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3234 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3235 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3239 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n"); in goya_validate_dma_pkt_host()
3241 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3242 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3248 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3250 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3251 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3254 dev_err(hdev->dev, "DMA direction is undefined\n"); in goya_validate_dma_pkt_host()
3255 return -EFAULT; in goya_validate_dma_pkt_host()
3260 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_host()
3261 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_host()
3262 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_host()
3264 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3267 user_dma_pkt->tsize); in goya_validate_dma_pkt_host()
3268 return -EFAULT; in goya_validate_dma_pkt_host()
3272 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_host()
3273 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_host()
3274 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_host()
3276 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3279 user_dma_pkt->tsize); in goya_validate_dma_pkt_host()
3280 return -EFAULT; in goya_validate_dma_pkt_host()
3285 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_host()
3288 (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) { in goya_validate_dma_pkt_host()
3289 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3291 return -EFAULT; in goya_validate_dma_pkt_host()
3309 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_no_host()
3314 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n"); in goya_validate_dma_pkt_no_host()
3315 dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_no_host()
3316 sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_no_host()
3318 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n"); in goya_validate_dma_pkt_no_host()
3319 sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_no_host()
3320 dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_no_host()
3324 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_no_host()
3325 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_no_host()
3326 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_no_host()
3327 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3328 sram_memory_addr, user_dma_pkt->tsize); in goya_validate_dma_pkt_no_host()
3329 return -EFAULT; in goya_validate_dma_pkt_no_host()
3333 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_no_host()
3334 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_no_host()
3335 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_no_host()
3336 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3337 dram_memory_addr, user_dma_pkt->tsize); in goya_validate_dma_pkt_no_host()
3338 return -EFAULT; in goya_validate_dma_pkt_no_host()
3341 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_no_host()
3354 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_no_mmu()
3355 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3356 le64_to_cpu(user_dma_pkt->src_addr)); in goya_validate_dma_pkt_no_mmu()
3357 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3358 le64_to_cpu(user_dma_pkt->dst_addr)); in goya_validate_dma_pkt_no_mmu()
3359 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_no_mmu()
3361 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_no_mmu()
3369 if (user_dma_pkt->tsize == 0) { in goya_validate_dma_pkt_no_mmu()
3370 dev_err(hdev->dev, in goya_validate_dma_pkt_no_mmu()
3372 return -EINVAL; in goya_validate_dma_pkt_no_mmu()
3387 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_mmu()
3388 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3389 le64_to_cpu(user_dma_pkt->src_addr)); in goya_validate_dma_pkt_mmu()
3390 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3391 le64_to_cpu(user_dma_pkt->dst_addr)); in goya_validate_dma_pkt_mmu()
3392 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_mmu()
3395 * WA for HW-23. in goya_validate_dma_pkt_mmu()
3399 if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 && in goya_validate_dma_pkt_mmu()
3400 hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr), in goya_validate_dma_pkt_mmu()
3401 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_mmu()
3402 hdev->asic_prop.pmmu.start_addr, in goya_validate_dma_pkt_mmu()
3403 hdev->asic_prop.pmmu.end_addr)) { in goya_validate_dma_pkt_mmu()
3404 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3406 return -EFAULT; in goya_validate_dma_pkt_mmu()
3409 if (user_dma_pkt->tsize == 0) { in goya_validate_dma_pkt_mmu()
3410 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3412 return -EINVAL; in goya_validate_dma_pkt_mmu()
3415 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_mmu()
3424 struct goya_device *goya = hdev->asic_specific; in goya_validate_wreg32()
3428 reg_offset = le32_to_cpu(wreg_pkt->ctl) & in goya_validate_wreg32()
3431 dev_dbg(hdev->dev, "WREG32 packet details:\n"); in goya_validate_wreg32()
3432 dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset); in goya_validate_wreg32()
3433 dev_dbg(hdev->dev, "value == 0x%x\n", in goya_validate_wreg32()
3434 le32_to_cpu(wreg_pkt->value)); in goya_validate_wreg32()
3437 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n", in goya_validate_wreg32()
3439 return -EPERM; in goya_validate_wreg32()
3443 * With MMU, DMA channels are not secured, so it doesn't matter where in goya_validate_wreg32()
3445 * non-secured property in goya_validate_wreg32()
3447 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_validate_wreg32()
3453 if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) || in goya_validate_wreg32()
3454 (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) { in goya_validate_wreg32()
3456 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n", in goya_validate_wreg32()
3457 wreg_pkt->value); in goya_validate_wreg32()
3458 return -EPERM; in goya_validate_wreg32()
3470 parser->patched_cb_size = 0; in goya_validate_cb()
3473 while (cb_parsed_length < parser->user_cb_size) { in goya_validate_cb()
3478 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in goya_validate_cb()
3481 (le64_to_cpu(user_pkt->header) & in goya_validate_cb()
3486 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_validate_cb()
3487 rc = -EINVAL; in goya_validate_cb()
3493 if (cb_parsed_length > parser->user_cb_size) { in goya_validate_cb()
3494 dev_err(hdev->dev, in goya_validate_cb()
3496 rc = -EINVAL; in goya_validate_cb()
3505 * not called in MMU path while this function is called in goya_validate_cb()
3509 parser->patched_cb_size += pkt_size; in goya_validate_cb()
3513 dev_err(hdev->dev, in goya_validate_cb()
3515 rc = -EPERM; in goya_validate_cb()
3519 dev_err(hdev->dev, in goya_validate_cb()
3521 rc = -EPERM; in goya_validate_cb()
3525 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_validate_cb()
3526 rc = -EPERM; in goya_validate_cb()
3530 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_validate_cb()
3531 rc = -EPERM; in goya_validate_cb()
3547 parser->patched_cb_size += pkt_size; in goya_validate_cb()
3551 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_validate_cb()
3553 rc = -EINVAL; in goya_validate_cb()
3564 * 2. A packet that will generate MSI-X interrupt in goya_validate_cb()
3566 parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; in goya_validate_cb()
3590 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_patch_dma_packet()
3599 (user_dma_pkt->tsize == 0)) { in goya_patch_dma_packet()
3606 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_patch_dma_packet()
3607 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_patch_dma_packet()
3612 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_patch_dma_packet()
3613 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_patch_dma_packet()
3619 le32_to_cpu(user_dma_pkt->tsize), in goya_patch_dma_packet()
3620 parser->job_userptr_list, &userptr) == false)) { in goya_patch_dma_packet()
3621 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", in goya_patch_dma_packet()
3622 addr, user_dma_pkt->tsize); in goya_patch_dma_packet()
3623 return -EFAULT; in goya_patch_dma_packet()
3636 sgt = userptr->sgt; in goya_patch_dma_packet()
3639 for_each_sg(sgt->sgl, sg, sgt->nents, count) { in goya_patch_dma_packet()
3646 while ((count + 1) < sgt->nents) { in goya_patch_dma_packet()
3664 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_patch_dma_packet()
3669 new_dma_pkt->ctl = cpu_to_le32(ctl); in goya_patch_dma_packet()
3670 new_dma_pkt->tsize = cpu_to_le32((u32) len); in goya_patch_dma_packet()
3673 new_dma_pkt->src_addr = cpu_to_le64(dma_addr); in goya_patch_dma_packet()
3674 new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr); in goya_patch_dma_packet()
3676 new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr); in goya_patch_dma_packet()
3677 new_dma_pkt->dst_addr = cpu_to_le64(dma_addr); in goya_patch_dma_packet()
3687 dev_err(hdev->dev, in goya_patch_dma_packet()
3689 return -EFAULT; in goya_patch_dma_packet()
3692 /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */ in goya_patch_dma_packet()
3693 new_dma_pkt--; in goya_patch_dma_packet()
3694 new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask); in goya_patch_dma_packet()
3709 while (cb_parsed_length < parser->user_cb_size) { in goya_patch_cb()
3715 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in goya_patch_cb()
3716 kernel_pkt = parser->patched_cb->kernel_address + in goya_patch_cb()
3720 (le64_to_cpu(user_pkt->header) & in goya_patch_cb()
3725 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_patch_cb()
3726 rc = -EINVAL; in goya_patch_cb()
3732 if (cb_parsed_length > parser->user_cb_size) { in goya_patch_cb()
3733 dev_err(hdev->dev, in goya_patch_cb()
3735 rc = -EINVAL; in goya_patch_cb()
3756 dev_err(hdev->dev, in goya_patch_cb()
3758 rc = -EPERM; in goya_patch_cb()
3762 dev_err(hdev->dev, in goya_patch_cb()
3764 rc = -EPERM; in goya_patch_cb()
3768 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_patch_cb()
3769 rc = -EPERM; in goya_patch_cb()
3773 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_patch_cb()
3774 rc = -EPERM; in goya_patch_cb()
3786 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_patch_cb()
3788 rc = -EINVAL; in goya_patch_cb()
3810 * 2. A packet that will generate MSI-X interrupt in goya_parse_cb_mmu()
3812 parser->patched_cb_size = parser->user_cb_size + in goya_parse_cb_mmu()
3815 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, in goya_parse_cb_mmu()
3816 parser->patched_cb_size, false, false, in goya_parse_cb_mmu()
3820 dev_err(hdev->dev, in goya_parse_cb_mmu()
3827 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3830 WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", in goya_parse_cb_mmu()
3832 if (!parser->patched_cb) { in goya_parse_cb_mmu()
3833 rc = -EFAULT; in goya_parse_cb_mmu()
3838 * The check that parser->user_cb_size <= parser->user_cb->size was done in goya_parse_cb_mmu()
3841 memcpy(parser->patched_cb->kernel_address, in goya_parse_cb_mmu()
3842 parser->user_cb->kernel_address, in goya_parse_cb_mmu()
3843 parser->user_cb_size); in goya_parse_cb_mmu()
3845 patched_cb_size = parser->patched_cb_size; in goya_parse_cb_mmu()
3848 user_cb = parser->user_cb; in goya_parse_cb_mmu()
3849 parser->user_cb = parser->patched_cb; in goya_parse_cb_mmu()
3851 parser->user_cb = user_cb; in goya_parse_cb_mmu()
3854 hl_cb_put(parser->patched_cb); in goya_parse_cb_mmu()
3858 if (patched_cb_size != parser->patched_cb_size) { in goya_parse_cb_mmu()
3859 dev_err(hdev->dev, "user CB size mismatch\n"); in goya_parse_cb_mmu()
3860 hl_cb_put(parser->patched_cb); in goya_parse_cb_mmu()
3861 rc = -EINVAL; in goya_parse_cb_mmu()
3872 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3889 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, in goya_parse_cb_no_mmu()
3890 parser->patched_cb_size, false, false, in goya_parse_cb_no_mmu()
3893 dev_err(hdev->dev, in goya_parse_cb_no_mmu()
3899 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
3902 WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", in goya_parse_cb_no_mmu()
3904 if (!parser->patched_cb) { in goya_parse_cb_no_mmu()
3905 rc = -EFAULT; in goya_parse_cb_no_mmu()
3912 hl_cb_put(parser->patched_cb); in goya_parse_cb_no_mmu()
3921 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
3926 hl_userptr_delete_list(hdev, parser->job_userptr_list); in goya_parse_cb_no_mmu()
3933 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in goya_parse_cb_no_ext_queue()
3934 struct goya_device *goya = hdev->asic_specific; in goya_parse_cb_no_ext_queue()
3936 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_parse_cb_no_ext_queue()
3941 (u64) (uintptr_t) parser->user_cb, in goya_parse_cb_no_ext_queue()
3942 parser->user_cb_size, in goya_parse_cb_no_ext_queue()
3943 asic_prop->sram_user_base_address, in goya_parse_cb_no_ext_queue()
3944 asic_prop->sram_end_address)) in goya_parse_cb_no_ext_queue()
3948 (u64) (uintptr_t) parser->user_cb, in goya_parse_cb_no_ext_queue()
3949 parser->user_cb_size, in goya_parse_cb_no_ext_queue()
3950 asic_prop->dram_user_base_address, in goya_parse_cb_no_ext_queue()
3951 asic_prop->dram_end_address)) in goya_parse_cb_no_ext_queue()
3954 dev_err(hdev->dev, in goya_parse_cb_no_ext_queue()
3956 parser->user_cb, parser->user_cb_size); in goya_parse_cb_no_ext_queue()
3958 return -EFAULT; in goya_parse_cb_no_ext_queue()
3963 struct goya_device *goya = hdev->asic_specific; in goya_cs_parser()
3965 if (parser->queue_type == QUEUE_TYPE_INT) in goya_cs_parser()
3968 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_cs_parser()
3981 cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2); in goya_add_end_of_cb_packets()
3986 cq_pkt->ctl = cpu_to_le32(tmp); in goya_add_end_of_cb_packets()
3987 cq_pkt->value = cpu_to_le32(cq_val); in goya_add_end_of_cb_packets()
3988 cq_pkt->addr = cpu_to_le64(cq_addr); in goya_add_end_of_cb_packets()
3994 cq_pkt->ctl = cpu_to_le32(tmp); in goya_add_end_of_cb_packets()
3995 cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF); in goya_add_end_of_cb_packets()
3996 cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF); in goya_add_end_of_cb_packets()
4014 ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4); in goya_clear_sm_regs()
4017 ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4); in goya_clear_sm_regs()
4030 * goya_debugfs_read32 - read a 32bit value from a given device or a host mapped
4046 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_read32()
4051 *val = RREG32(addr - CFG_BASE); in goya_debugfs_read32()
4056 *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_read32()
4057 (addr - SRAM_BASE_ADDR)); in goya_debugfs_read32()
4059 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) { in goya_debugfs_read32()
4062 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_read32()
4066 *val = readl(hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_read32()
4067 (addr - bar_base_addr)); in goya_debugfs_read32()
4073 rc = -EIO; in goya_debugfs_read32()
4076 *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE); in goya_debugfs_read32()
4079 rc = -EFAULT; in goya_debugfs_read32()
4086 * goya_debugfs_write32 - write a 32bit value to a given device or a host mapped
4102 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_write32()
4107 WREG32(addr - CFG_BASE, val); in goya_debugfs_write32()
4112 writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_write32()
4113 (addr - SRAM_BASE_ADDR)); in goya_debugfs_write32()
4115 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) { in goya_debugfs_write32()
4118 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_write32()
4122 writel(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_write32()
4123 (addr - bar_base_addr)); in goya_debugfs_write32()
4129 rc = -EIO; in goya_debugfs_write32()
4132 *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; in goya_debugfs_write32()
4135 rc = -EFAULT; in goya_debugfs_write32()
4143 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_read64()
4147 if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) { in goya_debugfs_read64()
4148 u32 val_l = RREG32(addr - CFG_BASE); in goya_debugfs_read64()
4149 u32 val_h = RREG32(addr + sizeof(u32) - CFG_BASE); in goya_debugfs_read64()
4154 (addr <= SRAM_BASE_ADDR + SRAM_SIZE - sizeof(u64))) { in goya_debugfs_read64()
4156 *val = readq(hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_read64()
4157 (addr - SRAM_BASE_ADDR)); in goya_debugfs_read64()
4160 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) { in goya_debugfs_read64()
4163 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_read64()
4167 *val = readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_read64()
4168 (addr - bar_base_addr)); in goya_debugfs_read64()
4174 rc = -EIO; in goya_debugfs_read64()
4177 *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE); in goya_debugfs_read64()
4180 rc = -EFAULT; in goya_debugfs_read64()
4188 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_write64()
4192 if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) { in goya_debugfs_write64()
4193 WREG32(addr - CFG_BASE, lower_32_bits(val)); in goya_debugfs_write64()
4194 WREG32(addr + sizeof(u32) - CFG_BASE, upper_32_bits(val)); in goya_debugfs_write64()
4197 (addr <= SRAM_BASE_ADDR + SRAM_SIZE - sizeof(u64))) { in goya_debugfs_write64()
4199 writeq(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_write64()
4200 (addr - SRAM_BASE_ADDR)); in goya_debugfs_write64()
4203 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) { in goya_debugfs_write64()
4206 (addr & ~(prop->dram_pci_bar_size - 0x1ull)); in goya_debugfs_write64()
4210 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_write64()
4211 (addr - bar_base_addr)); in goya_debugfs_write64()
4217 rc = -EIO; in goya_debugfs_write64()
4220 *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; in goya_debugfs_write64()
4223 rc = -EFAULT; in goya_debugfs_write64()
4231 struct goya_device *goya = hdev->asic_specific; in goya_read_pte()
4233 if (hdev->hard_reset_pending) in goya_read_pte()
4236 return readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_read_pte()
4237 (addr - goya->ddr_bar_cur_addr)); in goya_read_pte()
4242 struct goya_device *goya = hdev->asic_specific; in goya_write_pte()
4244 if (hdev->hard_reset_pending) in goya_write_pte()
4247 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_write_pte()
4248 (addr - goya->ddr_bar_cur_addr)); in goya_write_pte()
4374 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3; in goya_get_event_desc()
4378 index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0; in goya_get_event_desc()
4382 index = event_type - GOYA_ASYNC_EVENT_ID_PLL0; in goya_get_event_desc()
4393 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_DEC) / 3; in goya_get_event_desc()
4404 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR) / 10; in goya_get_event_desc()
4408 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_CMDQ; in goya_get_event_desc()
4412 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_QM; in goya_get_event_desc()
4416 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_QM; in goya_get_event_desc()
4420 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH; in goya_get_event_desc()
4431 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10; in goya_get_event_desc()
4435 index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0; in goya_get_event_desc()
4447 dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n"); in goya_print_razwi_info()
4452 dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n"); in goya_print_razwi_info()
4457 dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n"); in goya_print_razwi_info()
4462 dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n"); in goya_print_razwi_info()
4469 struct goya_device *goya = hdev->asic_specific; in goya_print_mmu_error_info()
4473 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_print_mmu_error_info()
4482 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", in goya_print_mmu_error_info()
4495 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in goya_print_irq_info()
4517 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */ in goya_unmask_irq_arr()
4522 dev_err(hdev->dev, "too many elements in IRQ array\n"); in goya_unmask_irq_arr()
4523 return -EINVAL; in goya_unmask_irq_arr()
4528 return -ENOMEM; in goya_unmask_irq_arr()
4531 pkt->length = cpu_to_le32(irq_num_entries); in goya_unmask_irq_arr()
4536 for (irq_arr_index = 0, goya_irq_arr = (__le32 *) &pkt->irqs; in goya_unmask_irq_arr()
4541 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY << in goya_unmask_irq_arr()
4544 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt, in goya_unmask_irq_arr()
4548 dev_err(hdev->dev, "failed to unmask IRQ array\n"); in goya_unmask_irq_arr()
4577 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), in goya_unmask_irq()
4581 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type); in goya_unmask_irq()
4590 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4591 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4595 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4596 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4600 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4601 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4605 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4606 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4611 dev_err(hdev->dev, "Received invalid clock change event %d\n", in goya_print_clk_change_info()
4619 u32 ctl = le32_to_cpu(eq_entry->hdr.ctl); in goya_handle_eqe()
4622 struct goya_device *goya = hdev->asic_specific; in goya_handle_eqe()
4624 goya->events_stat[event_type]++; in goya_handle_eqe()
4625 goya->events_stat_aggregate[event_type]++; in goya_handle_eqe()
4652 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4710 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", in goya_handle_eqe()
4718 struct goya_device *goya = hdev->asic_specific; in goya_get_events_stat()
4721 *size = (u32) sizeof(goya->events_stat_aggregate); in goya_get_events_stat()
4722 return goya->events_stat_aggregate; in goya_get_events_stat()
4725 *size = (u32) sizeof(goya->events_stat); in goya_get_events_stat()
4726 return goya->events_stat; in goya_get_events_stat()
4743 return -ENOMEM; in goya_memset_device_memory()
4745 lin_dma_pkt = cb->kernel_address; in goya_memset_device_memory()
4757 lin_dma_pkt->ctl = cpu_to_le32(ctl); in goya_memset_device_memory()
4759 lin_dma_pkt->src_addr = cpu_to_le64(val); in goya_memset_device_memory()
4760 lin_dma_pkt->dst_addr = cpu_to_le64(addr); in goya_memset_device_memory()
4762 lin_dma_pkt->tsize = cpu_to_le32(SZ_2G); in goya_memset_device_memory()
4764 lin_dma_pkt->tsize = cpu_to_le32(size); in goya_memset_device_memory()
4766 size -= SZ_2G; in goya_memset_device_memory()
4769 } while (--lin_dma_pkts_cnt); in goya_memset_device_memory()
4773 dev_err(hdev->dev, "Failed to allocate a new job\n"); in goya_memset_device_memory()
4774 rc = -ENOMEM; in goya_memset_device_memory()
4778 job->id = 0; in goya_memset_device_memory()
4779 job->user_cb = cb; in goya_memset_device_memory()
4780 job->user_cb->cs_cnt++; in goya_memset_device_memory()
4781 job->user_cb_size = cb_size; in goya_memset_device_memory()
4782 job->hw_queue_id = GOYA_QUEUE_ID_DMA_0; in goya_memset_device_memory()
4783 job->patched_cb = job->user_cb; in goya_memset_device_memory()
4784 job->job_cb_size = job->user_cb_size; in goya_memset_device_memory()
4792 cb->cs_cnt--; in goya_memset_device_memory()
4796 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); in goya_memset_device_memory()
4803 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_context_switch()
4804 u64 addr = prop->sram_base_address, sob_addr; in goya_context_switch()
4805 u32 size = hdev->pldm ? 0x10000 : prop->sram_size; in goya_context_switch()
4808 u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO - in goya_context_switch()
4813 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n"); in goya_context_switch()
4823 (dma_id - 1) * 4; in goya_context_switch()
4839 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_clear_pgt_range()
4840 struct goya_device *goya = hdev->asic_specific; in goya_mmu_clear_pgt_range()
4841 u64 addr = prop->mmu_pgt_addr; in goya_mmu_clear_pgt_range()
4842 u32 size = prop->mmu_pgt_size + MMU_DRAM_DEFAULT_PAGE_SIZE + in goya_mmu_clear_pgt_range()
4845 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_clear_pgt_range()
4853 struct goya_device *goya = hdev->asic_specific; in goya_mmu_set_dram_default_page()
4854 u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; in goya_mmu_set_dram_default_page()
4858 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_set_dram_default_page()
4866 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_add_mappings_for_device_cpu()
4867 struct goya_device *goya = hdev->asic_specific; in goya_mmu_add_mappings_for_device_cpu()
4871 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_add_mappings_for_device_cpu()
4875 rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off, in goya_mmu_add_mappings_for_device_cpu()
4876 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_add_mappings_for_device_cpu()
4879 dev_err(hdev->dev, "Map failed for address 0x%llx\n", in goya_mmu_add_mappings_for_device_cpu()
4880 prop->dram_base_address + off); in goya_mmu_add_mappings_for_device_cpu()
4885 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_add_mappings_for_device_cpu()
4886 rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, in goya_mmu_add_mappings_for_device_cpu()
4887 hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB, true); in goya_mmu_add_mappings_for_device_cpu()
4890 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4892 off -= PAGE_SIZE_2MB; in goya_mmu_add_mappings_for_device_cpu()
4897 rc = hl_mmu_map(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4899 hdev->cpu_accessible_dma_address + cpu_off, in goya_mmu_add_mappings_for_device_cpu()
4902 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4904 cpu_off -= PAGE_SIZE_4KB; in goya_mmu_add_mappings_for_device_cpu()
4918 goya->device_cpu_mmu_mappings_done = true; in goya_mmu_add_mappings_for_device_cpu()
4923 for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB) in goya_mmu_add_mappings_for_device_cpu()
4924 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4927 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4931 for (; off >= 0 ; off -= PAGE_SIZE_2MB) in goya_mmu_add_mappings_for_device_cpu()
4932 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4933 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_add_mappings_for_device_cpu()
4935 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4937 prop->dram_base_address + off); in goya_mmu_add_mappings_for_device_cpu()
4944 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_remove_device_cpu_mappings()
4945 struct goya_device *goya = hdev->asic_specific; in goya_mmu_remove_device_cpu_mappings()
4948 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_remove_device_cpu_mappings()
4951 if (!goya->device_cpu_mmu_mappings_done) in goya_mmu_remove_device_cpu_mappings()
4957 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_remove_device_cpu_mappings()
4958 if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, in goya_mmu_remove_device_cpu_mappings()
4960 dev_warn(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4964 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4968 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4974 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4975 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_remove_device_cpu_mappings()
4977 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4979 prop->dram_base_address + off); in goya_mmu_remove_device_cpu_mappings()
4981 goya->device_cpu_mmu_mappings_done = false; in goya_mmu_remove_device_cpu_mappings()
4986 struct goya_device *goya = hdev->asic_specific; in goya_mmu_prepare()
4989 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_prepare()
5005 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache()
5009 if (!(goya->hw_cap_initialized & HW_CAP_MMU) || in goya_mmu_invalidate_cache()
5010 hdev->hard_reset_pending) in goya_mmu_invalidate_cache()
5017 if (hdev->pldm) in goya_mmu_invalidate_cache()
5022 mutex_lock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache()
5035 mutex_unlock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache()
5038 dev_err_ratelimited(hdev->dev, in goya_mmu_invalidate_cache()
5039 "MMU cache invalidation timeout\n"); in goya_mmu_invalidate_cache()
5049 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache_range()
5053 if (!(goya->hw_cap_initialized & HW_CAP_MMU) || in goya_mmu_invalidate_cache_range()
5054 hdev->hard_reset_pending) in goya_mmu_invalidate_cache_range()
5061 if (hdev->pldm) in goya_mmu_invalidate_cache_range()
5066 mutex_lock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache_range()
5090 mutex_unlock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache_range()
5093 dev_err_ratelimited(hdev->dev, in goya_mmu_invalidate_cache_range()
5094 "MMU cache invalidation timeout\n"); in goya_mmu_invalidate_cache_range()
5103 struct goya_device *goya = hdev->asic_specific; in goya_send_heartbeat()
5105 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_send_heartbeat()
5113 struct goya_device *goya = hdev->asic_specific; in goya_cpucp_info_get()
5114 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_cpucp_info_get()
5118 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_cpucp_info_get()
5125 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); in goya_cpucp_info_get()
5129 dev_err(hdev->dev, in goya_cpucp_info_get()
5135 prop->dram_size = dram_size; in goya_cpucp_info_get()
5136 prop->dram_end_address = prop->dram_base_address + dram_size; in goya_cpucp_info_get()
5139 if (!strlen(prop->cpucp_info.card_name)) in goya_cpucp_info_get()
5140 strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, in goya_cpucp_info_get()
5159 const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; in goya_is_device_idle()
5160 const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; in goya_is_device_idle()
5169 "--- ------- ------------ -------------\n"); in goya_is_device_idle()
5171 offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0; in goya_is_device_idle()
5191 "--- ------- ------------ -------------- ----------\n"); in goya_is_device_idle()
5193 offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0; in goya_is_device_idle()
5215 "--- ------- ------------ -------------- -----------\n"); in goya_is_device_idle()
5237 __acquires(&goya->hw_queues_lock) in goya_hw_queues_lock()
5239 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_lock()
5241 spin_lock(&goya->hw_queues_lock); in goya_hw_queues_lock()
5245 __releases(&goya->hw_queues_lock) in goya_hw_queues_unlock()
5247 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_unlock()
5249 spin_unlock(&goya->hw_queues_lock); in goya_hw_queues_unlock()
5254 return hdev->pdev->device; in goya_get_pci_id()
5260 struct goya_device *goya = hdev->asic_specific; in goya_get_eeprom_data()
5262 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_get_eeprom_data()
5313 dev_dbg(hdev->dev, "Working in 64-bit DMA mode\n"); in goya_set_dma_mask_from_fw()
5314 hdev->power9_64bit_dma_enable = 1; in goya_set_dma_mask_from_fw()
5315 hdev->dma_mask = 64; in goya_set_dma_mask_from_fw()
5317 dev_dbg(hdev->dev, "Working in 48-bit DMA mode\n"); in goya_set_dma_mask_from_fw()
5318 hdev->power9_64bit_dma_enable = 0; in goya_set_dma_mask_from_fw()
5319 hdev->dma_mask = 48; in goya_set_dma_mask_from_fw()
5406 * goya_set_asic_funcs - set Goya function pointers
5413 hdev->asic_funcs = &goya_funcs; in goya_set_asic_funcs()