1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4 #include <linux/interrupt.h>
5 #include <linux/iopoll.h>
6 #include <linux/device.h>
7
8 #include "lima_device.h"
9 #include "lima_mmu.h"
10 #include "lima_vm.h"
11 #include "lima_object.h"
12 #include "lima_regs.h"
13
14 #define mmu_write(reg, data) writel(data, ip->iomem + reg)
15 #define mmu_read(reg) readl(ip->iomem + reg)
16
17 #define lima_mmu_send_command(cmd, addr, val, cond) \
18 ({ \
19 int __ret; \
20 \
21 mmu_write(LIMA_MMU_COMMAND, cmd); \
22 __ret = readl_poll_timeout(ip->iomem + (addr), val, \
23 cond, 0, 100); \
24 if (__ret) \
25 dev_err(dev->dev, \
26 "mmu command %x timeout\n", cmd); \
27 __ret; \
28 })
29
lima_mmu_irq_handler(int irq,void * data)30 static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
31 {
32 struct lima_ip *ip = data;
33 struct lima_device *dev = ip->dev;
34 u32 status = mmu_read(LIMA_MMU_INT_STATUS);
35 struct lima_sched_pipe *pipe;
36
37 /* for shared irq case */
38 if (!status)
39 return IRQ_NONE;
40
41 if (status & LIMA_MMU_INT_PAGE_FAULT) {
42 u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
43
44 dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n",
45 fault, LIMA_MMU_STATUS_BUS_ID(status),
46 status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read",
47 lima_ip_name(ip));
48 }
49
50 if (status & LIMA_MMU_INT_READ_BUS_ERROR)
51 dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip));
52
53 /* mask all interrupts before resume */
54 mmu_write(LIMA_MMU_INT_MASK, 0);
55 mmu_write(LIMA_MMU_INT_CLEAR, status);
56
57 pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
58 lima_sched_pipe_mmu_error(pipe);
59
60 return IRQ_HANDLED;
61 }
62
lima_mmu_init(struct lima_ip * ip)63 int lima_mmu_init(struct lima_ip *ip)
64 {
65 struct lima_device *dev = ip->dev;
66 int err;
67 u32 v;
68
69 if (ip->id == lima_ip_ppmmu_bcast)
70 return 0;
71
72 mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
73 if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
74 dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip));
75 return -EIO;
76 }
77
78 mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
79 err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
80 LIMA_MMU_DTE_ADDR, v, v == 0);
81 if (err)
82 return err;
83
84 err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
85 IRQF_SHARED, lima_ip_name(ip), ip);
86 if (err) {
87 dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip));
88 return err;
89 }
90
91 mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
92 mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
93 return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
94 LIMA_MMU_STATUS, v,
95 v & LIMA_MMU_STATUS_PAGING_ENABLED);
96 }
97
lima_mmu_fini(struct lima_ip * ip)98 void lima_mmu_fini(struct lima_ip *ip)
99 {
100
101 }
102
lima_mmu_switch_vm(struct lima_ip * ip,struct lima_vm * vm)103 void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
104 {
105 struct lima_device *dev = ip->dev;
106 u32 v;
107
108 lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
109 LIMA_MMU_STATUS, v,
110 v & LIMA_MMU_STATUS_STALL_ACTIVE);
111
112 if (vm)
113 mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
114
115 /* flush the TLB */
116 mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
117
118 lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
119 LIMA_MMU_STATUS, v,
120 !(v & LIMA_MMU_STATUS_STALL_ACTIVE));
121 }
122
lima_mmu_page_fault_resume(struct lima_ip * ip)123 void lima_mmu_page_fault_resume(struct lima_ip *ip)
124 {
125 struct lima_device *dev = ip->dev;
126 u32 status = mmu_read(LIMA_MMU_STATUS);
127 u32 v;
128
129 if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
130 dev_info(dev->dev, "mmu resume\n");
131
132 mmu_write(LIMA_MMU_INT_MASK, 0);
133 mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
134 lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
135 LIMA_MMU_DTE_ADDR, v, v == 0);
136 mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
137 mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
138 lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
139 LIMA_MMU_STATUS, v,
140 v & LIMA_MMU_STATUS_PAGING_ENABLED);
141 }
142 }
143
