243 static void ivpu_mmu_config_check(struct ivpu_device *vdev)  in ivpu_mmu_config_check()  argument
248 	if (ivpu_is_simics(vdev))  in ivpu_mmu_config_check()
255 		ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
259 		ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);  in ivpu_mmu_config_check()
263 		ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);  in ivpu_mmu_config_check()
265 	if (ivpu_is_simics(vdev))  in ivpu_mmu_config_check()
267 	else if (ivpu_is_fpga(vdev))  in ivpu_mmu_config_check()
274 		ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
277 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)  in ivpu_mmu_cdtab_alloc()  argument
279 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cdtab_alloc()
283 	cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);  in ivpu_mmu_cdtab_alloc()
287 	ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);  in ivpu_mmu_cdtab_alloc()
292 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)  in ivpu_mmu_strtab_alloc()  argument
294 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_alloc()
298 	strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);  in ivpu_mmu_strtab_alloc()
306 	ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",  in ivpu_mmu_strtab_alloc()
312 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_alloc()  argument
314 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cmdq_alloc()
317 	q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);  in ivpu_mmu_cmdq_alloc()
325 	ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_cmdq_alloc()
331 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)  in ivpu_mmu_evtq_alloc()  argument
333 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_evtq_alloc()
336 	q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);  in ivpu_mmu_evtq_alloc()
344 	ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_evtq_alloc()
350 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)  in ivpu_mmu_structs_alloc()  argument
354 	ret = ivpu_mmu_cdtab_alloc(vdev);  in ivpu_mmu_structs_alloc()
356 		ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);  in ivpu_mmu_structs_alloc()
360 	ret = ivpu_mmu_strtab_alloc(vdev);  in ivpu_mmu_structs_alloc()
362 		ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);  in ivpu_mmu_structs_alloc()
366 	ret = ivpu_mmu_cmdq_alloc(vdev);  in ivpu_mmu_structs_alloc()
368 		ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);  in ivpu_mmu_structs_alloc()
372 	ret = ivpu_mmu_evtq_alloc(vdev);  in ivpu_mmu_structs_alloc()
374 		ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);  in ivpu_mmu_structs_alloc()
379 static int ivpu_mmu_reg_write(struct ivpu_device *vdev, u32 reg, u32 val)  in ivpu_mmu_reg_write()  argument
389 		ivpu_err(vdev, "Failed to write register 0x%x\n", reg);  in ivpu_mmu_reg_write()
394 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)  in ivpu_mmu_irqs_setup()  argument
399 	ret = ivpu_mmu_reg_write(vdev, VPU_37XX_HOST_MMU_IRQ_CTRL, 0);  in ivpu_mmu_irqs_setup()
403 	return ivpu_mmu_reg_write(vdev, VPU_37XX_HOST_MMU_IRQ_CTRL, irq_ctrl);  in ivpu_mmu_irqs_setup()
406 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_wait_for_cons()  argument
408 	struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_wait_for_cons()
414 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)  in ivpu_mmu_cmdq_cmd_write()  argument
416 	struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_cmd_write()
421 		ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);  in ivpu_mmu_cmdq_cmd_write()
429 	ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);  in ivpu_mmu_cmdq_cmd_write()
434 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_sync()  argument
436 	struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_sync()
445 	ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);  in ivpu_mmu_cmdq_sync()
452 	ret = ivpu_mmu_cmdq_wait_for_cons(vdev);  in ivpu_mmu_cmdq_sync()
454 		ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret);  in ivpu_mmu_cmdq_sync()
459 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_write_cfgi_all()  argument
464 	return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);  in ivpu_mmu_cmdq_write_cfgi_all()
467 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)  in ivpu_mmu_cmdq_write_tlbi_nh_asid()  argument
472 	return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);  in ivpu_mmu_cmdq_write_tlbi_nh_asid()
475 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_write_tlbi_nsnh_all()  argument
479 	return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);  in ivpu_mmu_cmdq_write_tlbi_nsnh_all()
482 static int ivpu_mmu_reset(struct ivpu_device *vdev)  in ivpu_mmu_reset()  argument
484 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_reset()
498 	ret = ivpu_mmu_reg_write(vdev, VPU_37XX_HOST_MMU_CR0, 0);  in ivpu_mmu_reset()
518 	ret = ivpu_mmu_reg_write(vdev, VPU_37XX_HOST_MMU_CR0, val);  in ivpu_mmu_reset()
522 	ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);  in ivpu_mmu_reset()
526 	ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);  in ivpu_mmu_reset()
530 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_reset()
539 	ret = ivpu_mmu_reg_write(vdev, VPU_37XX_HOST_MMU_CR0, val);  in ivpu_mmu_reset()
544 	ret = ivpu_mmu_reg_write(vdev, VPU_37XX_HOST_MMU_CR0, val);  in ivpu_mmu_reset()
548 	ret = ivpu_mmu_irqs_setup(vdev);  in ivpu_mmu_reset()
553 	return ivpu_mmu_reg_write(vdev, VPU_37XX_HOST_MMU_CR0, val);  in ivpu_mmu_reset()
556 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)  in ivpu_mmu_strtab_link_cd()  argument
558 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_link_cd()
586 	ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);  in ivpu_mmu_strtab_link_cd()
589 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)  in ivpu_mmu_strtab_init()  argument
591 	ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);  in ivpu_mmu_strtab_init()
592 	ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);  in ivpu_mmu_strtab_init()
597 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)  in ivpu_mmu_invalidate_tlb()  argument
599 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_invalidate_tlb()
606 	ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);  in ivpu_mmu_invalidate_tlb()
610 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_invalidate_tlb()
616 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)  in ivpu_mmu_cd_add()  argument
618 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cd_add()
660 	ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",  in ivpu_mmu_cd_add()
667 	ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);  in ivpu_mmu_cd_add()
671 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_cd_add()
677 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev)  in ivpu_mmu_cd_add_gbl()  argument
681 	ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma);  in ivpu_mmu_cd_add_gbl()
683 		ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret);  in ivpu_mmu_cd_add_gbl()
688 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma)  in ivpu_mmu_cd_add_user()  argument
693 		ivpu_err(vdev, "Invalid SSID: %u\n", ssid);  in ivpu_mmu_cd_add_user()
697 	ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma);  in ivpu_mmu_cd_add_user()
699 		ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret);  in ivpu_mmu_cd_add_user()
704 int ivpu_mmu_init(struct ivpu_device *vdev)  in ivpu_mmu_init()  argument
706 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_init()
709 	ivpu_dbg(vdev, MMU, "Init..\n");  in ivpu_mmu_init()
711 	drmm_mutex_init(&vdev->drm, &mmu->lock);  in ivpu_mmu_init()
712 	ivpu_mmu_config_check(vdev);  in ivpu_mmu_init()
714 	ret = ivpu_mmu_structs_alloc(vdev);  in ivpu_mmu_init()
718 	ret = ivpu_mmu_strtab_init(vdev);  in ivpu_mmu_init()
720 		ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);  in ivpu_mmu_init()
724 	ret = ivpu_mmu_cd_add_gbl(vdev);  in ivpu_mmu_init()
726 		ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);  in ivpu_mmu_init()
730 	ret = ivpu_mmu_enable(vdev);  in ivpu_mmu_init()
732 		ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);  in ivpu_mmu_init()
736 	ivpu_dbg(vdev, MMU, "Init done\n");  in ivpu_mmu_init()
741 int ivpu_mmu_enable(struct ivpu_device *vdev)  in ivpu_mmu_enable()  argument
743 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_enable()
750 	ret = ivpu_mmu_reset(vdev);  in ivpu_mmu_enable()
752 		ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);  in ivpu_mmu_enable()
756 	ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);  in ivpu_mmu_enable()
760 	ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);  in ivpu_mmu_enable()
764 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_enable()
777 void ivpu_mmu_disable(struct ivpu_device *vdev)  in ivpu_mmu_disable()  argument
779 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_disable()
786 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)  in ivpu_mmu_dump_event()  argument
794 …ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch…  in ivpu_mmu_dump_event()
798 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)  in ivpu_mmu_get_event()  argument
800 	struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;  in ivpu_mmu_get_event()
816 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)  in ivpu_mmu_irq_evtq_handler()  argument
822 	ivpu_dbg(vdev, IRQ, "MMU event queue\n");  in ivpu_mmu_irq_evtq_handler()
824 	while ((event = ivpu_mmu_get_event(vdev)) != NULL) {  in ivpu_mmu_irq_evtq_handler()
825 		ivpu_mmu_dump_event(vdev, event);  in ivpu_mmu_irq_evtq_handler()
831 			ivpu_mmu_user_context_mark_invalid(vdev, ssid);  in ivpu_mmu_irq_evtq_handler()
835 		ivpu_pm_schedule_recovery(vdev);  in ivpu_mmu_irq_evtq_handler()
838 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)  in ivpu_mmu_irq_gerr_handler()  argument
842 	ivpu_dbg(vdev, IRQ, "MMU error\n");  in ivpu_mmu_irq_gerr_handler()
852 		ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
855 		ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
858 		ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
861 		ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
864 		ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
867 		ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
870 		ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
875 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_set_pgtable()  argument
877 	return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma);  in ivpu_mmu_set_pgtable()
880 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid)  in ivpu_mmu_clear_pgtable()  argument
882 	ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */  in ivpu_mmu_clear_pgtable()