1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright 2019 Collabora Ltd */
3
4 #include <drm/drm_file.h>
5 #include <drm/drm_gem_shmem_helper.h>
6 #include <drm/panfrost_drm.h>
7 #include <linux/completion.h>
8 #include <linux/iopoll.h>
9 #include <linux/pm_runtime.h>
10 #include <linux/slab.h>
11 #include <linux/uaccess.h>
12
13 #include "panfrost_device.h"
14 #include "panfrost_features.h"
15 #include "panfrost_gem.h"
16 #include "panfrost_issues.h"
17 #include "panfrost_job.h"
18 #include "panfrost_mmu.h"
19 #include "panfrost_perfcnt.h"
20 #include "panfrost_regs.h"
21
22 #define COUNTERS_PER_BLOCK 64
23 #define BYTES_PER_COUNTER 4
24 #define BLOCKS_PER_COREGROUP 8
25 #define V4_SHADERS_PER_COREGROUP 4
26
27 struct panfrost_perfcnt {
28 struct panfrost_gem_mapping *mapping;
29 size_t bosize;
30 void *buf;
31 struct panfrost_file_priv *user;
32 struct mutex lock;
33 struct completion dump_comp;
34 };
35
panfrost_perfcnt_clean_cache_done(struct panfrost_device * pfdev)36 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
37 {
38 complete(&pfdev->perfcnt->dump_comp);
39 }
40
panfrost_perfcnt_sample_done(struct panfrost_device * pfdev)41 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
42 {
43 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
44 }
45
panfrost_perfcnt_dump_locked(struct panfrost_device * pfdev)46 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
47 {
48 u64 gpuva;
49 int ret;
50
51 reinit_completion(&pfdev->perfcnt->dump_comp);
52 gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT;
53 gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
54 gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
55 gpu_write(pfdev, GPU_INT_CLEAR,
56 GPU_IRQ_CLEAN_CACHES_COMPLETED |
57 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
58 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
59 ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
60 msecs_to_jiffies(1000));
61 if (!ret)
62 ret = -ETIMEDOUT;
63 else if (ret > 0)
64 ret = 0;
65
66 return ret;
67 }
68
panfrost_perfcnt_enable_locked(struct panfrost_device * pfdev,struct drm_file * file_priv,unsigned int counterset)69 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
70 struct drm_file *file_priv,
71 unsigned int counterset)
72 {
73 struct panfrost_file_priv *user = file_priv->driver_priv;
74 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
75 struct drm_gem_shmem_object *bo;
76 u32 cfg, as;
77 int ret;
78
79 if (user == perfcnt->user)
80 return 0;
81 else if (perfcnt->user)
82 return -EBUSY;
83
84 ret = pm_runtime_get_sync(pfdev->dev);
85 if (ret < 0)
86 goto err_put_pm;
87
88 bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
89 if (IS_ERR(bo)) {
90 ret = PTR_ERR(bo);
91 goto err_put_pm;
92 }
93
94 /* Map the perfcnt buf in the address space attached to file_priv. */
95 ret = panfrost_gem_open(&bo->base, file_priv);
96 if (ret)
97 goto err_put_bo;
98
99 perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base),
100 user);
101 if (!perfcnt->mapping) {
102 ret = -EINVAL;
103 goto err_close_bo;
104 }
105
106 perfcnt->buf = drm_gem_shmem_vmap(&bo->base);
107 if (IS_ERR(perfcnt->buf)) {
108 ret = PTR_ERR(perfcnt->buf);
109 goto err_put_mapping;
110 }
111
112 /*
113 * Invalidate the cache and clear the counters to start from a fresh
114 * state.
115 */
116 reinit_completion(&pfdev->perfcnt->dump_comp);
117 gpu_write(pfdev, GPU_INT_CLEAR,
118 GPU_IRQ_CLEAN_CACHES_COMPLETED |
119 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
120 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
121 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
122 ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
123 msecs_to_jiffies(1000));
124 if (!ret) {
125 ret = -ETIMEDOUT;
126 goto err_vunmap;
127 }
128
129 perfcnt->user = user;
130
131 as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu);
132 cfg = GPU_PERFCNT_CFG_AS(as) |
133 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
134
135 /*
136 * Bifrost GPUs have 2 set of counters, but we're only interested by
137 * the first one for now.
138 */
139 if (panfrost_model_is_bifrost(pfdev))
140 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
141
142 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
143 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
144 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
145
146 /*
147 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
148 * counters.
149 */
150 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
151 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
152 else
153 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
154
155 gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
156
157 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
158 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
159
160 /* The BO ref is retained by the mapping. */
161 drm_gem_object_put(&bo->base);
162
163 return 0;
164
165 err_vunmap:
166 drm_gem_shmem_vunmap(&bo->base, perfcnt->buf);
167 err_put_mapping:
168 panfrost_gem_mapping_put(perfcnt->mapping);
169 err_close_bo:
170 panfrost_gem_close(&bo->base, file_priv);
171 err_put_bo:
172 drm_gem_object_put(&bo->base);
173 err_put_pm:
174 pm_runtime_put(pfdev->dev);
175 return ret;
176 }
177
panfrost_perfcnt_disable_locked(struct panfrost_device * pfdev,struct drm_file * file_priv)178 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
179 struct drm_file *file_priv)
180 {
181 struct panfrost_file_priv *user = file_priv->driver_priv;
182 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
183
184 if (user != perfcnt->user)
185 return -EINVAL;
186
187 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
188 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
189 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
190 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
191 gpu_write(pfdev, GPU_PERFCNT_CFG,
192 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
193
194 perfcnt->user = NULL;
195 drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, perfcnt->buf);
196 perfcnt->buf = NULL;
197 panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv);
198 panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu);
199 panfrost_gem_mapping_put(perfcnt->mapping);
200 perfcnt->mapping = NULL;
201 pm_runtime_mark_last_busy(pfdev->dev);
202 pm_runtime_put_autosuspend(pfdev->dev);
203
204 return 0;
205 }
206
panfrost_ioctl_perfcnt_enable(struct drm_device * dev,void * data,struct drm_file * file_priv)207 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
208 struct drm_file *file_priv)
209 {
210 struct panfrost_device *pfdev = dev->dev_private;
211 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
212 struct drm_panfrost_perfcnt_enable *req = data;
213 int ret;
214
215 ret = panfrost_unstable_ioctl_check();
216 if (ret)
217 return ret;
218
219 /* Only Bifrost GPUs have 2 set of counters. */
220 if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
221 return -EINVAL;
222
223 mutex_lock(&perfcnt->lock);
224 if (req->enable)
225 ret = panfrost_perfcnt_enable_locked(pfdev, file_priv,
226 req->counterset);
227 else
228 ret = panfrost_perfcnt_disable_locked(pfdev, file_priv);
229 mutex_unlock(&perfcnt->lock);
230
231 return ret;
232 }
233
panfrost_ioctl_perfcnt_dump(struct drm_device * dev,void * data,struct drm_file * file_priv)234 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
235 struct drm_file *file_priv)
236 {
237 struct panfrost_device *pfdev = dev->dev_private;
238 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
239 struct drm_panfrost_perfcnt_dump *req = data;
240 void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
241 int ret;
242
243 ret = panfrost_unstable_ioctl_check();
244 if (ret)
245 return ret;
246
247 mutex_lock(&perfcnt->lock);
248 if (perfcnt->user != file_priv->driver_priv) {
249 ret = -EINVAL;
250 goto out;
251 }
252
253 ret = panfrost_perfcnt_dump_locked(pfdev);
254 if (ret)
255 goto out;
256
257 if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
258 ret = -EFAULT;
259
260 out:
261 mutex_unlock(&perfcnt->lock);
262
263 return ret;
264 }
265
panfrost_perfcnt_close(struct drm_file * file_priv)266 void panfrost_perfcnt_close(struct drm_file *file_priv)
267 {
268 struct panfrost_file_priv *pfile = file_priv->driver_priv;
269 struct panfrost_device *pfdev = pfile->pfdev;
270 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
271
272 pm_runtime_get_sync(pfdev->dev);
273 mutex_lock(&perfcnt->lock);
274 if (perfcnt->user == pfile)
275 panfrost_perfcnt_disable_locked(pfdev, file_priv);
276 mutex_unlock(&perfcnt->lock);
277 pm_runtime_mark_last_busy(pfdev->dev);
278 pm_runtime_put_autosuspend(pfdev->dev);
279 }
280
panfrost_perfcnt_init(struct panfrost_device * pfdev)281 int panfrost_perfcnt_init(struct panfrost_device *pfdev)
282 {
283 struct panfrost_perfcnt *perfcnt;
284 size_t size;
285
286 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
287 unsigned int ncoregroups;
288
289 ncoregroups = hweight64(pfdev->features.l2_present);
290 size = ncoregroups * BLOCKS_PER_COREGROUP *
291 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
292 } else {
293 unsigned int nl2c, ncores;
294
295 /*
296 * TODO: define a macro to extract the number of l2 caches from
297 * mem_features.
298 */
299 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
300
301 /*
302 * shader_present might be sparse, but the counters layout
303 * forces to dump unused regions too, hence the fls64() call
304 * instead of hweight64().
305 */
306 ncores = fls64(pfdev->features.shader_present);
307
308 /*
309 * There's always one JM and one Tiler block, hence the '+ 2'
310 * here.
311 */
312 size = (nl2c + ncores + 2) *
313 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
314 }
315
316 perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
317 if (!perfcnt)
318 return -ENOMEM;
319
320 perfcnt->bosize = size;
321
322 /* Start with everything disabled. */
323 gpu_write(pfdev, GPU_PERFCNT_CFG,
324 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
325 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
326 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
327 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
328 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
329
330 init_completion(&perfcnt->dump_comp);
331 mutex_init(&perfcnt->lock);
332 pfdev->perfcnt = perfcnt;
333
334 return 0;
335 }
336
panfrost_perfcnt_fini(struct panfrost_device * pfdev)337 void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
338 {
339 /* Disable everything before leaving. */
340 gpu_write(pfdev, GPU_PERFCNT_CFG,
341 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
342 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
343 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
344 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
345 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
346 }
347
