1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2017 The Linux Foundation. All rights reserved.
3 */
4
5 #include "msm_gem.h"
6 #include "a5xx_gpu.h"
7
8 /*
9 * Try to transition the preemption state from old to new. Return
10 * true on success or false if the original state wasn't 'old'
11 */
try_preempt_state(struct a5xx_gpu * a5xx_gpu,enum preempt_state old,enum preempt_state new)12 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
13 enum preempt_state old, enum preempt_state new)
14 {
15 enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
16 old, new);
17
18 return (cur == old);
19 }
20
21 /*
22 * Force the preemption state to the specified state. This is used in cases
23 * where the current state is known and won't change
24 */
set_preempt_state(struct a5xx_gpu * gpu,enum preempt_state new)25 static inline void set_preempt_state(struct a5xx_gpu *gpu,
26 enum preempt_state new)
27 {
28 /*
29 * preempt_state may be read by other cores trying to trigger a
30 * preemption or in the interrupt handler so barriers are needed
31 * before...
32 */
33 smp_mb__before_atomic();
34 atomic_set(&gpu->preempt_state, new);
35 /* ... and after*/
36 smp_mb__after_atomic();
37 }
38
39 /* Write the most recent wptr for the given ring into the hardware */
update_wptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)40 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
41 {
42 unsigned long flags;
43 uint32_t wptr;
44
45 if (!ring)
46 return;
47
48 spin_lock_irqsave(&ring->lock, flags);
49 wptr = get_wptr(ring);
50 spin_unlock_irqrestore(&ring->lock, flags);
51
52 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
53 }
54
55 /* Return the highest priority ringbuffer with something in it */
get_next_ring(struct msm_gpu * gpu)56 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
57 {
58 unsigned long flags;
59 int i;
60
61 for (i = 0; i < gpu->nr_rings; i++) {
62 bool empty;
63 struct msm_ringbuffer *ring = gpu->rb[i];
64
65 spin_lock_irqsave(&ring->lock, flags);
66 empty = (get_wptr(ring) == ring->memptrs->rptr);
67 spin_unlock_irqrestore(&ring->lock, flags);
68
69 if (!empty)
70 return ring;
71 }
72
73 return NULL;
74 }
75
a5xx_preempt_timer(struct timer_list * t)76 static void a5xx_preempt_timer(struct timer_list *t)
77 {
78 struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
79 struct msm_gpu *gpu = &a5xx_gpu->base.base;
80 struct drm_device *dev = gpu->dev;
81 struct msm_drm_private *priv = dev->dev_private;
82
83 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
84 return;
85
86 DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
87 queue_work(priv->wq, &gpu->recover_work);
88 }
89
90 /* Try to trigger a preemption switch */
a5xx_preempt_trigger(struct msm_gpu * gpu)91 void a5xx_preempt_trigger(struct msm_gpu *gpu)
92 {
93 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
94 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
95 unsigned long flags;
96 struct msm_ringbuffer *ring;
97
98 if (gpu->nr_rings == 1)
99 return;
100
101 /*
102 * Try to start preemption by moving from NONE to START. If
103 * unsuccessful, a preemption is already in flight
104 */
105 if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
106 return;
107
108 /* Get the next ring to preempt to */
109 ring = get_next_ring(gpu);
110
111 /*
112 * If no ring is populated or the highest priority ring is the current
113 * one do nothing except to update the wptr to the latest and greatest
114 */
115 if (!ring || (a5xx_gpu->cur_ring == ring)) {
116 /*
117 * Its possible that while a preemption request is in progress
118 * from an irq context, a user context trying to submit might
119 * fail to update the write pointer, because it determines
120 * that the preempt state is not PREEMPT_NONE.
121 *
122 * Close the race by introducing an intermediate
123 * state PREEMPT_ABORT to let the submit path
124 * know that the ringbuffer is not going to change
125 * and can safely update the write pointer.
126 */
127
128 set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
129 update_wptr(gpu, a5xx_gpu->cur_ring);
130 set_preempt_state(a5xx_gpu, PREEMPT_NONE);
131 return;
132 }
133
134 /* Make sure the wptr doesn't update while we're in motion */
135 spin_lock_irqsave(&ring->lock, flags);
136 a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
137 spin_unlock_irqrestore(&ring->lock, flags);
138
139 /* Set the address of the incoming preemption record */
140 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
141 REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
142 a5xx_gpu->preempt_iova[ring->id]);
143
144 a5xx_gpu->next_ring = ring;
145
146 /* Start a timer to catch a stuck preemption */
147 mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
148
149 /* Set the preemption state to triggered */
150 set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
151
152 /* Make sure everything is written before hitting the button */
153 wmb();
154
155 /* And actually start the preemption */
156 gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
157 }
158
a5xx_preempt_irq(struct msm_gpu * gpu)159 void a5xx_preempt_irq(struct msm_gpu *gpu)
160 {
161 uint32_t status;
162 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
163 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
164 struct drm_device *dev = gpu->dev;
165 struct msm_drm_private *priv = dev->dev_private;
166
167 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
168 return;
169
170 /* Delete the preemption watchdog timer */
171 del_timer(&a5xx_gpu->preempt_timer);
172
173 /*
174 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
175 * firing the interrupt, but there is a non zero chance of a hardware
176 * condition or a software race that could set it again before we have a
177 * chance to finish. If that happens, log and go for recovery
178 */
179 status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
180 if (unlikely(status)) {
181 set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
182 DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
183 gpu->name);
184 queue_work(priv->wq, &gpu->recover_work);
185 return;
186 }
187
188 a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
189 a5xx_gpu->next_ring = NULL;
190
191 update_wptr(gpu, a5xx_gpu->cur_ring);
192
193 set_preempt_state(a5xx_gpu, PREEMPT_NONE);
194 }
195
a5xx_preempt_hw_init(struct msm_gpu * gpu)196 void a5xx_preempt_hw_init(struct msm_gpu *gpu)
197 {
198 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
199 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
200 int i;
201
202 /* Always come up on rb 0 */
203 a5xx_gpu->cur_ring = gpu->rb[0];
204
205 /* No preemption if we only have one ring */
206 if (gpu->nr_rings == 1)
207 return;
208
209 for (i = 0; i < gpu->nr_rings; i++) {
210 a5xx_gpu->preempt[i]->wptr = 0;
211 a5xx_gpu->preempt[i]->rptr = 0;
212 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
213 }
214
215 /* Write a 0 to signal that we aren't switching pagetables */
216 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
217 REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
218
219 /* Reset the preemption state */
220 set_preempt_state(a5xx_gpu, PREEMPT_NONE);
221 }
222
preempt_init_ring(struct a5xx_gpu * a5xx_gpu,struct msm_ringbuffer * ring)223 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
224 struct msm_ringbuffer *ring)
225 {
226 struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
227 struct msm_gpu *gpu = &adreno_gpu->base;
228 struct a5xx_preempt_record *ptr;
229 struct drm_gem_object *bo = NULL;
230 u64 iova = 0;
231
232 ptr = msm_gem_kernel_new(gpu->dev,
233 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
234 MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);
235
236 if (IS_ERR(ptr))
237 return PTR_ERR(ptr);
238
239 msm_gem_object_set_name(bo, "preempt");
240
241 a5xx_gpu->preempt_bo[ring->id] = bo;
242 a5xx_gpu->preempt_iova[ring->id] = iova;
243 a5xx_gpu->preempt[ring->id] = ptr;
244
245 /* Set up the defaults on the preemption record */
246
247 ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
248 ptr->info = 0;
249 ptr->data = 0;
250 ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
251 ptr->rptr_addr = rbmemptr(ring, rptr);
252 ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE;
253
254 return 0;
255 }
256
a5xx_preempt_fini(struct msm_gpu * gpu)257 void a5xx_preempt_fini(struct msm_gpu *gpu)
258 {
259 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
260 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
261 int i;
262
263 for (i = 0; i < gpu->nr_rings; i++)
264 msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace, true);
265 }
266
a5xx_preempt_init(struct msm_gpu * gpu)267 void a5xx_preempt_init(struct msm_gpu *gpu)
268 {
269 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
270 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
271 int i;
272
273 /* No preemption if we only have one ring */
274 if (gpu->nr_rings <= 1)
275 return;
276
277 for (i = 0; i < gpu->nr_rings; i++) {
278 if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
279 /*
280 * On any failure our adventure is over. Clean up and
281 * set nr_rings to 1 to force preemption off
282 */
283 a5xx_preempt_fini(gpu);
284 gpu->nr_rings = 1;
285
286 return;
287 }
288 }
289
290 timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
291 }
292
