1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 */
6
7 #include <linux/coresight.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/iommu.h>
10 #include <linux/slab.h>
11 #include <linux/vmalloc.h>
12 #include "coresight-catu.h"
13 #include "coresight-priv.h"
14 #include "coresight-tmc.h"
15
16 struct etr_flat_buf {
17 struct device *dev;
18 dma_addr_t daddr;
19 void *vaddr;
20 size_t size;
21 };
22
23 /*
24 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
25 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
26 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
27 * contain more than one SG buffer and tables.
28 *
29 * A table entry has the following format:
30 *
31 * ---Bit31------------Bit4-------Bit1-----Bit0--
32 * | Address[39:12] | SBZ | Entry Type |
33 * ----------------------------------------------
34 *
35 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
36 * always zero.
37 *
38 * Entry type:
39 * b00 - Reserved.
40 * b01 - Last entry in the tables, points to 4K page buffer.
41 * b10 - Normal entry, points to 4K page buffer.
42 * b11 - Link. The address points to the base of next table.
43 */
44
45 typedef u32 sgte_t;
46
47 #define ETR_SG_PAGE_SHIFT 12
48 #define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
49 #define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
50 #define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
51 #define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
52
53 #define ETR_SG_ET_MASK 0x3
54 #define ETR_SG_ET_LAST 0x1
55 #define ETR_SG_ET_NORMAL 0x2
56 #define ETR_SG_ET_LINK 0x3
57
58 #define ETR_SG_ADDR_SHIFT 4
59
60 #define ETR_SG_ENTRY(addr, type) \
61 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
62 (type & ETR_SG_ET_MASK))
63
64 #define ETR_SG_ADDR(entry) \
65 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
66 #define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
67
68 /*
69 * struct etr_sg_table : ETR SG Table
70 * @sg_table: Generic SG Table holding the data/table pages.
71 * @hwaddr: hwaddress used by the TMC, which is the base
72 * address of the table.
73 */
74 struct etr_sg_table {
75 struct tmc_sg_table *sg_table;
76 dma_addr_t hwaddr;
77 };
78
79 /*
80 * tmc_etr_sg_table_entries: Total number of table entries required to map
81 * @nr_pages system pages.
82 *
83 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
84 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
85 * with the last entry pointing to another page of table entries.
86 * If we spill over to a new page for mapping 1 entry, we could as
87 * well replace the link entry of the previous page with the last entry.
88 */
89 static inline unsigned long __attribute_const__
tmc_etr_sg_table_entries(int nr_pages)90 tmc_etr_sg_table_entries(int nr_pages)
91 {
92 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
93 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
94 /*
95 * If we spill over to a new page for 1 entry, we could as well
96 * make it the LAST entry in the previous page, skipping the Link
97 * address.
98 */
99 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
100 nr_sglinks--;
101 return nr_sgpages + nr_sglinks;
102 }
103
104 /*
105 * tmc_pages_get_offset: Go through all the pages in the tmc_pages
106 * and map the device address @addr to an offset within the virtual
107 * contiguous buffer.
108 */
109 static long
tmc_pages_get_offset(struct tmc_pages * tmc_pages,dma_addr_t addr)110 tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
111 {
112 int i;
113 dma_addr_t page_start;
114
115 for (i = 0; i < tmc_pages->nr_pages; i++) {
116 page_start = tmc_pages->daddrs[i];
117 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
118 return i * PAGE_SIZE + (addr - page_start);
119 }
120
121 return -EINVAL;
122 }
123
124 /*
125 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
126 * If the pages were not allocated in tmc_pages_alloc(), we would
127 * simply drop the refcount.
128 */
tmc_pages_free(struct tmc_pages * tmc_pages,struct device * dev,enum dma_data_direction dir)129 static void tmc_pages_free(struct tmc_pages *tmc_pages,
130 struct device *dev, enum dma_data_direction dir)
131 {
132 int i;
133
134 for (i = 0; i < tmc_pages->nr_pages; i++) {
135 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
136 dma_unmap_page(dev, tmc_pages->daddrs[i],
137 PAGE_SIZE, dir);
138 if (tmc_pages->pages && tmc_pages->pages[i])
139 __free_page(tmc_pages->pages[i]);
140 }
141
142 kfree(tmc_pages->pages);
143 kfree(tmc_pages->daddrs);
144 tmc_pages->pages = NULL;
145 tmc_pages->daddrs = NULL;
146 tmc_pages->nr_pages = 0;
147 }
148
149 /*
150 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
151 * If @pages is not NULL, the list of page virtual addresses are
152 * used as the data pages. The pages are then dma_map'ed for @dev
153 * with dma_direction @dir.
154 *
155 * Returns 0 upon success, else the error number.
156 */
tmc_pages_alloc(struct tmc_pages * tmc_pages,struct device * dev,int node,enum dma_data_direction dir,void ** pages)157 static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
158 struct device *dev, int node,
159 enum dma_data_direction dir, void **pages)
160 {
161 int i, nr_pages;
162 dma_addr_t paddr;
163 struct page *page;
164
165 nr_pages = tmc_pages->nr_pages;
166 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
167 GFP_KERNEL);
168 if (!tmc_pages->daddrs)
169 return -ENOMEM;
170 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
171 GFP_KERNEL);
172 if (!tmc_pages->pages) {
173 kfree(tmc_pages->daddrs);
174 tmc_pages->daddrs = NULL;
175 return -ENOMEM;
176 }
177
178 for (i = 0; i < nr_pages; i++) {
179 if (pages && pages[i]) {
180 page = virt_to_page(pages[i]);
181 /* Hold a refcount on the page */
182 get_page(page);
183 } else {
184 page = alloc_pages_node(node,
185 GFP_KERNEL | __GFP_ZERO, 0);
186 }
187 paddr = dma_map_page(dev, page, 0, PAGE_SIZE, dir);
188 if (dma_mapping_error(dev, paddr))
189 goto err;
190 tmc_pages->daddrs[i] = paddr;
191 tmc_pages->pages[i] = page;
192 }
193 return 0;
194 err:
195 tmc_pages_free(tmc_pages, dev, dir);
196 return -ENOMEM;
197 }
198
199 static inline long
tmc_sg_get_data_page_offset(struct tmc_sg_table * sg_table,dma_addr_t addr)200 tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
201 {
202 return tmc_pages_get_offset(&sg_table->data_pages, addr);
203 }
204
tmc_free_table_pages(struct tmc_sg_table * sg_table)205 static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
206 {
207 if (sg_table->table_vaddr)
208 vunmap(sg_table->table_vaddr);
209 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
210 }
211
tmc_free_data_pages(struct tmc_sg_table * sg_table)212 static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
213 {
214 if (sg_table->data_vaddr)
215 vunmap(sg_table->data_vaddr);
216 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
217 }
218
tmc_free_sg_table(struct tmc_sg_table * sg_table)219 void tmc_free_sg_table(struct tmc_sg_table *sg_table)
220 {
221 tmc_free_table_pages(sg_table);
222 tmc_free_data_pages(sg_table);
223 }
224
225 /*
226 * Alloc pages for the table. Since this will be used by the device,
227 * allocate the pages closer to the device (i.e, dev_to_node(dev)
228 * rather than the CPU node).
229 */
tmc_alloc_table_pages(struct tmc_sg_table * sg_table)230 static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
231 {
232 int rc;
233 struct tmc_pages *table_pages = &sg_table->table_pages;
234
235 rc = tmc_pages_alloc(table_pages, sg_table->dev,
236 dev_to_node(sg_table->dev),
237 DMA_TO_DEVICE, NULL);
238 if (rc)
239 return rc;
240 sg_table->table_vaddr = vmap(table_pages->pages,
241 table_pages->nr_pages,
242 VM_MAP,
243 PAGE_KERNEL);
244 if (!sg_table->table_vaddr)
245 rc = -ENOMEM;
246 else
247 sg_table->table_daddr = table_pages->daddrs[0];
248 return rc;
249 }
250
tmc_alloc_data_pages(struct tmc_sg_table * sg_table,void ** pages)251 static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
252 {
253 int rc;
254
255 /* Allocate data pages on the node requested by the caller */
256 rc = tmc_pages_alloc(&sg_table->data_pages,
257 sg_table->dev, sg_table->node,
258 DMA_FROM_DEVICE, pages);
259 if (!rc) {
260 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
261 sg_table->data_pages.nr_pages,
262 VM_MAP,
263 PAGE_KERNEL);
264 if (!sg_table->data_vaddr)
265 rc = -ENOMEM;
266 }
267 return rc;
268 }
269
270 /*
271 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
272 * and data buffers. TMC writes to the data buffers and reads from the SG
273 * Table pages.
274 *
275 * @dev - Device to which page should be DMA mapped.
276 * @node - Numa node for mem allocations
277 * @nr_tpages - Number of pages for the table entries.
278 * @nr_dpages - Number of pages for Data buffer.
279 * @pages - Optional list of virtual address of pages.
280 */
tmc_alloc_sg_table(struct device * dev,int node,int nr_tpages,int nr_dpages,void ** pages)281 struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
282 int node,
283 int nr_tpages,
284 int nr_dpages,
285 void **pages)
286 {
287 long rc;
288 struct tmc_sg_table *sg_table;
289
290 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
291 if (!sg_table)
292 return ERR_PTR(-ENOMEM);
293 sg_table->data_pages.nr_pages = nr_dpages;
294 sg_table->table_pages.nr_pages = nr_tpages;
295 sg_table->node = node;
296 sg_table->dev = dev;
297
298 rc = tmc_alloc_data_pages(sg_table, pages);
299 if (!rc)
300 rc = tmc_alloc_table_pages(sg_table);
301 if (rc) {
302 tmc_free_sg_table(sg_table);
303 kfree(sg_table);
304 return ERR_PTR(rc);
305 }
306
307 return sg_table;
308 }
309
310 /*
311 * tmc_sg_table_sync_data_range: Sync the data buffer written
312 * by the device from @offset upto a @size bytes.
313 */
tmc_sg_table_sync_data_range(struct tmc_sg_table * table,u64 offset,u64 size)314 void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
315 u64 offset, u64 size)
316 {
317 int i, index, start;
318 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
319 struct device *dev = table->dev;
320 struct tmc_pages *data = &table->data_pages;
321
322 start = offset >> PAGE_SHIFT;
323 for (i = start; i < (start + npages); i++) {
324 index = i % data->nr_pages;
325 dma_sync_single_for_cpu(dev, data->daddrs[index],
326 PAGE_SIZE, DMA_FROM_DEVICE);
327 }
328 }
329
330 /* tmc_sg_sync_table: Sync the page table */
tmc_sg_table_sync_table(struct tmc_sg_table * sg_table)331 void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
332 {
333 int i;
334 struct device *dev = sg_table->dev;
335 struct tmc_pages *table_pages = &sg_table->table_pages;
336
337 for (i = 0; i < table_pages->nr_pages; i++)
338 dma_sync_single_for_device(dev, table_pages->daddrs[i],
339 PAGE_SIZE, DMA_TO_DEVICE);
340 }
341
342 /*
343 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
344 * in the SG buffer. The @bufpp is updated to point to the buffer.
345 * Returns :
346 * the length of linear data available at @offset.
347 * or
348 * <= 0 if no data is available.
349 */
tmc_sg_table_get_data(struct tmc_sg_table * sg_table,u64 offset,size_t len,char ** bufpp)350 ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
351 u64 offset, size_t len, char **bufpp)
352 {
353 size_t size;
354 int pg_idx = offset >> PAGE_SHIFT;
355 int pg_offset = offset & (PAGE_SIZE - 1);
356 struct tmc_pages *data_pages = &sg_table->data_pages;
357
358 size = tmc_sg_table_buf_size(sg_table);
359 if (offset >= size)
360 return -EINVAL;
361
362 /* Make sure we don't go beyond the end */
363 len = (len < (size - offset)) ? len : size - offset;
364 /* Respect the page boundaries */
365 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
366 if (len > 0)
367 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
368 return len;
369 }
370
371 #ifdef ETR_SG_DEBUG
372 /* Map a dma address to virtual address */
373 static unsigned long
tmc_sg_daddr_to_vaddr(struct tmc_sg_table * sg_table,dma_addr_t addr,bool table)374 tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
375 dma_addr_t addr, bool table)
376 {
377 long offset;
378 unsigned long base;
379 struct tmc_pages *tmc_pages;
380
381 if (table) {
382 tmc_pages = &sg_table->table_pages;
383 base = (unsigned long)sg_table->table_vaddr;
384 } else {
385 tmc_pages = &sg_table->data_pages;
386 base = (unsigned long)sg_table->data_vaddr;
387 }
388
389 offset = tmc_pages_get_offset(tmc_pages, addr);
390 if (offset < 0)
391 return 0;
392 return base + offset;
393 }
394
395 /* Dump the given sg_table */
tmc_etr_sg_table_dump(struct etr_sg_table * etr_table)396 static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
397 {
398 sgte_t *ptr;
399 int i = 0;
400 dma_addr_t addr;
401 struct tmc_sg_table *sg_table = etr_table->sg_table;
402
403 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
404 etr_table->hwaddr, true);
405 while (ptr) {
406 addr = ETR_SG_ADDR(*ptr);
407 switch (ETR_SG_ET(*ptr)) {
408 case ETR_SG_ET_NORMAL:
409 dev_dbg(sg_table->dev,
410 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
411 ptr++;
412 break;
413 case ETR_SG_ET_LINK:
414 dev_dbg(sg_table->dev,
415 "%05d: *** %p\t:{L} 0x%llx ***\n",
416 i, ptr, addr);
417 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
418 addr, true);
419 break;
420 case ETR_SG_ET_LAST:
421 dev_dbg(sg_table->dev,
422 "%05d: ### %p\t:[L] 0x%llx ###\n",
423 i, ptr, addr);
424 return;
425 default:
426 dev_dbg(sg_table->dev,
427 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
428 i, ptr, addr);
429 return;
430 }
431 i++;
432 }
433 dev_dbg(sg_table->dev, "******* End of Table *****\n");
434 }
435 #else
tmc_etr_sg_table_dump(struct etr_sg_table * etr_table)436 static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
437 #endif
438
439 /*
440 * Populate the SG Table page table entries from table/data
441 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
442 * So does a Table page. So we keep track of indices of the tables
443 * in each system page and move the pointers accordingly.
444 */
445 #define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
tmc_etr_sg_table_populate(struct etr_sg_table * etr_table)446 static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
447 {
448 dma_addr_t paddr;
449 int i, type, nr_entries;
450 int tpidx = 0; /* index to the current system table_page */
451 int sgtidx = 0; /* index to the sg_table within the current syspage */
452 int sgtentry = 0; /* the entry within the sg_table */
453 int dpidx = 0; /* index to the current system data_page */
454 int spidx = 0; /* index to the SG page within the current data page */
455 sgte_t *ptr; /* pointer to the table entry to fill */
456 struct tmc_sg_table *sg_table = etr_table->sg_table;
457 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
458 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
459
460 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
461 /*
462 * Use the contiguous virtual address of the table to update entries.
463 */
464 ptr = sg_table->table_vaddr;
465 /*
466 * Fill all the entries, except the last entry to avoid special
467 * checks within the loop.
468 */
469 for (i = 0; i < nr_entries - 1; i++) {
470 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
471 /*
472 * Last entry in a sg_table page is a link address to
473 * the next table page. If this sg_table is the last
474 * one in the system page, it links to the first
475 * sg_table in the next system page. Otherwise, it
476 * links to the next sg_table page within the system
477 * page.
478 */
479 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
480 paddr = table_daddrs[tpidx + 1];
481 } else {
482 paddr = table_daddrs[tpidx] +
483 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
484 }
485 type = ETR_SG_ET_LINK;
486 } else {
487 /*
488 * Update the indices to the data_pages to point to the
489 * next sg_page in the data buffer.
490 */
491 type = ETR_SG_ET_NORMAL;
492 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
493 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
494 dpidx++;
495 }
496 *ptr++ = ETR_SG_ENTRY(paddr, type);
497 /*
498 * Move to the next table pointer, moving the table page index
499 * if necessary
500 */
501 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
502 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
503 tpidx++;
504 }
505 }
506
507 /* Set up the last entry, which is always a data pointer */
508 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
509 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
510 }
511
512 /*
513 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
514 * populate the table.
515 *
516 * @dev - Device pointer for the TMC
517 * @node - NUMA node where the memory should be allocated
518 * @size - Total size of the data buffer
519 * @pages - Optional list of page virtual address
520 */
521 static struct etr_sg_table *
tmc_init_etr_sg_table(struct device * dev,int node,unsigned long size,void ** pages)522 tmc_init_etr_sg_table(struct device *dev, int node,
523 unsigned long size, void **pages)
524 {
525 int nr_entries, nr_tpages;
526 int nr_dpages = size >> PAGE_SHIFT;
527 struct tmc_sg_table *sg_table;
528 struct etr_sg_table *etr_table;
529
530 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
531 if (!etr_table)
532 return ERR_PTR(-ENOMEM);
533 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
534 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
535
536 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
537 if (IS_ERR(sg_table)) {
538 kfree(etr_table);
539 return ERR_PTR(PTR_ERR(sg_table));
540 }
541
542 etr_table->sg_table = sg_table;
543 /* TMC should use table base address for DBA */
544 etr_table->hwaddr = sg_table->table_daddr;
545 tmc_etr_sg_table_populate(etr_table);
546 /* Sync the table pages for the HW */
547 tmc_sg_table_sync_table(sg_table);
548 tmc_etr_sg_table_dump(etr_table);
549
550 return etr_table;
551 }
552
553 /*
554 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
555 */
tmc_etr_alloc_flat_buf(struct tmc_drvdata * drvdata,struct etr_buf * etr_buf,int node,void ** pages)556 static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
557 struct etr_buf *etr_buf, int node,
558 void **pages)
559 {
560 struct etr_flat_buf *flat_buf;
561
562 /* We cannot reuse existing pages for flat buf */
563 if (pages)
564 return -EINVAL;
565
566 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
567 if (!flat_buf)
568 return -ENOMEM;
569
570 flat_buf->vaddr = dma_alloc_coherent(drvdata->dev, etr_buf->size,
571 &flat_buf->daddr, GFP_KERNEL);
572 if (!flat_buf->vaddr) {
573 kfree(flat_buf);
574 return -ENOMEM;
575 }
576
577 flat_buf->size = etr_buf->size;
578 flat_buf->dev = drvdata->dev;
579 etr_buf->hwaddr = flat_buf->daddr;
580 etr_buf->mode = ETR_MODE_FLAT;
581 etr_buf->private = flat_buf;
582 return 0;
583 }
584
tmc_etr_free_flat_buf(struct etr_buf * etr_buf)585 static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
586 {
587 struct etr_flat_buf *flat_buf = etr_buf->private;
588
589 if (flat_buf && flat_buf->daddr)
590 dma_free_coherent(flat_buf->dev, flat_buf->size,
591 flat_buf->vaddr, flat_buf->daddr);
592 kfree(flat_buf);
593 }
594
tmc_etr_sync_flat_buf(struct etr_buf * etr_buf,u64 rrp,u64 rwp)595 static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
596 {
597 /*
598 * Adjust the buffer to point to the beginning of the trace data
599 * and update the available trace data.
600 */
601 etr_buf->offset = rrp - etr_buf->hwaddr;
602 if (etr_buf->full)
603 etr_buf->len = etr_buf->size;
604 else
605 etr_buf->len = rwp - rrp;
606 }
607
tmc_etr_get_data_flat_buf(struct etr_buf * etr_buf,u64 offset,size_t len,char ** bufpp)608 static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
609 u64 offset, size_t len, char **bufpp)
610 {
611 struct etr_flat_buf *flat_buf = etr_buf->private;
612
613 *bufpp = (char *)flat_buf->vaddr + offset;
614 /*
615 * tmc_etr_buf_get_data already adjusts the length to handle
616 * buffer wrapping around.
617 */
618 return len;
619 }
620
621 static const struct etr_buf_operations etr_flat_buf_ops = {
622 .alloc = tmc_etr_alloc_flat_buf,
623 .free = tmc_etr_free_flat_buf,
624 .sync = tmc_etr_sync_flat_buf,
625 .get_data = tmc_etr_get_data_flat_buf,
626 };
627
628 /*
629 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
630 * appropriately.
631 */
tmc_etr_alloc_sg_buf(struct tmc_drvdata * drvdata,struct etr_buf * etr_buf,int node,void ** pages)632 static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
633 struct etr_buf *etr_buf, int node,
634 void **pages)
635 {
636 struct etr_sg_table *etr_table;
637
638 etr_table = tmc_init_etr_sg_table(drvdata->dev, node,
639 etr_buf->size, pages);
640 if (IS_ERR(etr_table))
641 return -ENOMEM;
642 etr_buf->hwaddr = etr_table->hwaddr;
643 etr_buf->mode = ETR_MODE_ETR_SG;
644 etr_buf->private = etr_table;
645 return 0;
646 }
647
tmc_etr_free_sg_buf(struct etr_buf * etr_buf)648 static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
649 {
650 struct etr_sg_table *etr_table = etr_buf->private;
651
652 if (etr_table) {
653 tmc_free_sg_table(etr_table->sg_table);
654 kfree(etr_table);
655 }
656 }
657
tmc_etr_get_data_sg_buf(struct etr_buf * etr_buf,u64 offset,size_t len,char ** bufpp)658 static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
659 size_t len, char **bufpp)
660 {
661 struct etr_sg_table *etr_table = etr_buf->private;
662
663 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
664 }
665
tmc_etr_sync_sg_buf(struct etr_buf * etr_buf,u64 rrp,u64 rwp)666 static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
667 {
668 long r_offset, w_offset;
669 struct etr_sg_table *etr_table = etr_buf->private;
670 struct tmc_sg_table *table = etr_table->sg_table;
671
672 /* Convert hw address to offset in the buffer */
673 r_offset = tmc_sg_get_data_page_offset(table, rrp);
674 if (r_offset < 0) {
675 dev_warn(table->dev,
676 "Unable to map RRP %llx to offset\n", rrp);
677 etr_buf->len = 0;
678 return;
679 }
680
681 w_offset = tmc_sg_get_data_page_offset(table, rwp);
682 if (w_offset < 0) {
683 dev_warn(table->dev,
684 "Unable to map RWP %llx to offset\n", rwp);
685 etr_buf->len = 0;
686 return;
687 }
688
689 etr_buf->offset = r_offset;
690 if (etr_buf->full)
691 etr_buf->len = etr_buf->size;
692 else
693 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
694 w_offset - r_offset;
695 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
696 }
697
698 static const struct etr_buf_operations etr_sg_buf_ops = {
699 .alloc = tmc_etr_alloc_sg_buf,
700 .free = tmc_etr_free_sg_buf,
701 .sync = tmc_etr_sync_sg_buf,
702 .get_data = tmc_etr_get_data_sg_buf,
703 };
704
705 /*
706 * TMC ETR could be connected to a CATU device, which can provide address
707 * translation service. This is represented by the Output port of the TMC
708 * (ETR) connected to the input port of the CATU.
709 *
710 * Returns : coresight_device ptr for the CATU device if a CATU is found.
711 * : NULL otherwise.
712 */
713 struct coresight_device *
tmc_etr_get_catu_device(struct tmc_drvdata * drvdata)714 tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
715 {
716 int i;
717 struct coresight_device *tmp, *etr = drvdata->csdev;
718
719 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
720 return NULL;
721
722 for (i = 0; i < etr->nr_outport; i++) {
723 tmp = etr->conns[i].child_dev;
724 if (tmp && coresight_is_catu_device(tmp))
725 return tmp;
726 }
727
728 return NULL;
729 }
730
tmc_etr_enable_catu(struct tmc_drvdata * drvdata)731 static inline void tmc_etr_enable_catu(struct tmc_drvdata *drvdata)
732 {
733 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
734
735 if (catu && helper_ops(catu)->enable)
736 helper_ops(catu)->enable(catu, drvdata->etr_buf);
737 }
738
tmc_etr_disable_catu(struct tmc_drvdata * drvdata)739 static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
740 {
741 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
742
743 if (catu && helper_ops(catu)->disable)
744 helper_ops(catu)->disable(catu, drvdata->etr_buf);
745 }
746
747 static const struct etr_buf_operations *etr_buf_ops[] = {
748 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
749 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
750 [ETR_MODE_CATU] = &etr_catu_buf_ops,
751 };
752
tmc_etr_mode_alloc_buf(int mode,struct tmc_drvdata * drvdata,struct etr_buf * etr_buf,int node,void ** pages)753 static inline int tmc_etr_mode_alloc_buf(int mode,
754 struct tmc_drvdata *drvdata,
755 struct etr_buf *etr_buf, int node,
756 void **pages)
757 {
758 int rc = -EINVAL;
759
760 switch (mode) {
761 case ETR_MODE_FLAT:
762 case ETR_MODE_ETR_SG:
763 case ETR_MODE_CATU:
764 if (etr_buf_ops[mode]->alloc)
765 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
766 node, pages);
767 if (!rc)
768 etr_buf->ops = etr_buf_ops[mode];
769 return rc;
770 default:
771 return -EINVAL;
772 }
773 }
774
775 /*
776 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
777 * @drvdata : ETR device details.
778 * @size : size of the requested buffer.
779 * @flags : Required properties for the buffer.
780 * @node : Node for memory allocations.
781 * @pages : An optional list of pages.
782 */
tmc_alloc_etr_buf(struct tmc_drvdata * drvdata,ssize_t size,int flags,int node,void ** pages)783 static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
784 ssize_t size, int flags,
785 int node, void **pages)
786 {
787 int rc = -ENOMEM;
788 bool has_etr_sg, has_iommu;
789 bool has_sg, has_catu;
790 struct etr_buf *etr_buf;
791
792 has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
793 has_iommu = iommu_get_domain_for_dev(drvdata->dev);
794 has_catu = !!tmc_etr_get_catu_device(drvdata);
795
796 has_sg = has_catu || has_etr_sg;
797
798 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
799 if (!etr_buf)
800 return ERR_PTR(-ENOMEM);
801
802 etr_buf->size = size;
803
804 /*
805 * If we have to use an existing list of pages, we cannot reliably
806 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
807 * we use the contiguous DMA memory if at least one of the following
808 * conditions is true:
809 * a) The ETR cannot use Scatter-Gather.
810 * b) we have a backing IOMMU
811 * c) The requested memory size is smaller (< 1M).
812 *
813 * Fallback to available mechanisms.
814 *
815 */
816 if (!pages &&
817 (!has_sg || has_iommu || size < SZ_1M))
818 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
819 etr_buf, node, pages);
820 if (rc && has_etr_sg)
821 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
822 etr_buf, node, pages);
823 if (rc && has_catu)
824 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
825 etr_buf, node, pages);
826 if (rc) {
827 kfree(etr_buf);
828 return ERR_PTR(rc);
829 }
830
831 dev_dbg(drvdata->dev, "allocated buffer of size %ldKB in mode %d\n",
832 (unsigned long)size >> 10, etr_buf->mode);
833 return etr_buf;
834 }
835
tmc_free_etr_buf(struct etr_buf * etr_buf)836 static void tmc_free_etr_buf(struct etr_buf *etr_buf)
837 {
838 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
839 etr_buf->ops->free(etr_buf);
840 kfree(etr_buf);
841 }
842
843 /*
844 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
845 * with a maximum of @len bytes.
846 * Returns: The size of the linear data available @pos, with *bufpp
847 * updated to point to the buffer.
848 */
tmc_etr_buf_get_data(struct etr_buf * etr_buf,u64 offset,size_t len,char ** bufpp)849 static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
850 u64 offset, size_t len, char **bufpp)
851 {
852 /* Adjust the length to limit this transaction to end of buffer */
853 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
854
855 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
856 }
857
858 static inline s64
tmc_etr_buf_insert_barrier_packet(struct etr_buf * etr_buf,u64 offset)859 tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
860 {
861 ssize_t len;
862 char *bufp;
863
864 len = tmc_etr_buf_get_data(etr_buf, offset,
865 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
866 if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
867 return -EINVAL;
868 coresight_insert_barrier_packet(bufp);
869 return offset + CORESIGHT_BARRIER_PKT_SIZE;
870 }
871
872 /*
873 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
874 * Makes sure the trace data is synced to the memory for consumption.
875 * @etr_buf->offset will hold the offset to the beginning of the trace data
876 * within the buffer, with @etr_buf->len bytes to consume.
877 */
tmc_sync_etr_buf(struct tmc_drvdata * drvdata)878 static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
879 {
880 struct etr_buf *etr_buf = drvdata->etr_buf;
881 u64 rrp, rwp;
882 u32 status;
883
884 rrp = tmc_read_rrp(drvdata);
885 rwp = tmc_read_rwp(drvdata);
886 status = readl_relaxed(drvdata->base + TMC_STS);
887 etr_buf->full = status & TMC_STS_FULL;
888
889 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
890
891 etr_buf->ops->sync(etr_buf, rrp, rwp);
892
893 /* Insert barrier packets at the beginning, if there was an overflow */
894 if (etr_buf->full)
895 tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
896 }
897
tmc_etr_enable_hw(struct tmc_drvdata * drvdata)898 static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
899 {
900 u32 axictl, sts;
901 struct etr_buf *etr_buf = drvdata->etr_buf;
902
903 /*
904 * If this ETR is connected to a CATU, enable it before we turn
905 * this on
906 */
907 tmc_etr_enable_catu(drvdata);
908
909 CS_UNLOCK(drvdata->base);
910
911 /* Wait for TMCSReady bit to be set */
912 tmc_wait_for_tmcready(drvdata);
913
914 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
915 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
916
917 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
918 axictl &= ~TMC_AXICTL_CLEAR_MASK;
919 axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16);
920 axictl |= TMC_AXICTL_AXCACHE_OS;
921
922 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
923 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
924 axictl |= TMC_AXICTL_ARCACHE_OS;
925 }
926
927 if (etr_buf->mode == ETR_MODE_ETR_SG) {
928 if (WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
929 return;
930 axictl |= TMC_AXICTL_SCT_GAT_MODE;
931 }
932
933 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
934 tmc_write_dba(drvdata, etr_buf->hwaddr);
935 /*
936 * If the TMC pointers must be programmed before the session,
937 * we have to set it properly (i.e, RRP/RWP to base address and
938 * STS to "not full").
939 */
940 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
941 tmc_write_rrp(drvdata, etr_buf->hwaddr);
942 tmc_write_rwp(drvdata, etr_buf->hwaddr);
943 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
944 writel_relaxed(sts, drvdata->base + TMC_STS);
945 }
946
947 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
948 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
949 TMC_FFCR_TRIGON_TRIGIN,
950 drvdata->base + TMC_FFCR);
951 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
952 tmc_enable_hw(drvdata);
953
954 CS_LOCK(drvdata->base);
955 }
956
957 /*
958 * Return the available trace data in the buffer (starts at etr_buf->offset,
959 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
960 * also updating the @bufpp on where to find it. Since the trace data
961 * starts at anywhere in the buffer, depending on the RRP, we adjust the
962 * @len returned to handle buffer wrapping around.
963 */
tmc_etr_get_sysfs_trace(struct tmc_drvdata * drvdata,loff_t pos,size_t len,char ** bufpp)964 ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
965 loff_t pos, size_t len, char **bufpp)
966 {
967 s64 offset;
968 ssize_t actual = len;
969 struct etr_buf *etr_buf = drvdata->etr_buf;
970
971 if (pos + actual > etr_buf->len)
972 actual = etr_buf->len - pos;
973 if (actual <= 0)
974 return actual;
975
976 /* Compute the offset from which we read the data */
977 offset = etr_buf->offset + pos;
978 if (offset >= etr_buf->size)
979 offset -= etr_buf->size;
980 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
981 }
982
983 static struct etr_buf *
tmc_etr_setup_sysfs_buf(struct tmc_drvdata * drvdata)984 tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
985 {
986 return tmc_alloc_etr_buf(drvdata, drvdata->size,
987 0, cpu_to_node(0), NULL);
988 }
989
990 static void
tmc_etr_free_sysfs_buf(struct etr_buf * buf)991 tmc_etr_free_sysfs_buf(struct etr_buf *buf)
992 {
993 if (buf)
994 tmc_free_etr_buf(buf);
995 }
996
tmc_etr_sync_sysfs_buf(struct tmc_drvdata * drvdata)997 static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
998 {
999 tmc_sync_etr_buf(drvdata);
1000 }
1001
tmc_etr_disable_hw(struct tmc_drvdata * drvdata)1002 static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1003 {
1004 CS_UNLOCK(drvdata->base);
1005
1006 tmc_flush_and_stop(drvdata);
1007 /*
1008 * When operating in sysFS mode the content of the buffer needs to be
1009 * read before the TMC is disabled.
1010 */
1011 if (drvdata->mode == CS_MODE_SYSFS)
1012 tmc_etr_sync_sysfs_buf(drvdata);
1013
1014 tmc_disable_hw(drvdata);
1015
1016 CS_LOCK(drvdata->base);
1017
1018 /* Disable CATU device if this ETR is connected to one */
1019 tmc_etr_disable_catu(drvdata);
1020 }
1021
tmc_enable_etr_sink_sysfs(struct coresight_device * csdev)1022 static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1023 {
1024 int ret = 0;
1025 unsigned long flags;
1026 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1027 struct etr_buf *new_buf = NULL, *free_buf = NULL;
1028
1029 /*
1030 * If we are enabling the ETR from disabled state, we need to make
1031 * sure we have a buffer with the right size. The etr_buf is not reset
1032 * immediately after we stop the tracing in SYSFS mode as we wait for
1033 * the user to collect the data. We may be able to reuse the existing
1034 * buffer, provided the size matches. Any allocation has to be done
1035 * with the lock released.
1036 */
1037 spin_lock_irqsave(&drvdata->spinlock, flags);
1038 if (!drvdata->etr_buf || (drvdata->etr_buf->size != drvdata->size)) {
1039 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1040
1041 /* Allocate memory with the locks released */
1042 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1043 if (IS_ERR(new_buf))
1044 return PTR_ERR(new_buf);
1045
1046 /* Let's try again */
1047 spin_lock_irqsave(&drvdata->spinlock, flags);
1048 }
1049
1050 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1051 ret = -EBUSY;
1052 goto out;
1053 }
1054
1055 /*
1056 * In sysFS mode we can have multiple writers per sink. Since this
1057 * sink is already enabled no memory is needed and the HW need not be
1058 * touched, even if the buffer size has changed.
1059 */
1060 if (drvdata->mode == CS_MODE_SYSFS)
1061 goto out;
1062
1063 /*
1064 * If we don't have a buffer or it doesn't match the requested size,
1065 * use the buffer allocated above. Otherwise reuse the existing buffer.
1066 */
1067 if (!drvdata->etr_buf ||
1068 (new_buf && drvdata->etr_buf->size != new_buf->size)) {
1069 free_buf = drvdata->etr_buf;
1070 drvdata->etr_buf = new_buf;
1071 }
1072
1073 drvdata->mode = CS_MODE_SYSFS;
1074 tmc_etr_enable_hw(drvdata);
1075 out:
1076 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1077
1078 /* Free memory outside the spinlock if need be */
1079 if (free_buf)
1080 tmc_etr_free_sysfs_buf(free_buf);
1081
1082 if (!ret)
1083 dev_info(drvdata->dev, "TMC-ETR enabled\n");
1084
1085 return ret;
1086 }
1087
tmc_enable_etr_sink_perf(struct coresight_device * csdev)1088 static int tmc_enable_etr_sink_perf(struct coresight_device *csdev)
1089 {
1090 /* We don't support perf mode yet ! */
1091 return -EINVAL;
1092 }
1093
tmc_enable_etr_sink(struct coresight_device * csdev,u32 mode)1094 static int tmc_enable_etr_sink(struct coresight_device *csdev, u32 mode)
1095 {
1096 switch (mode) {
1097 case CS_MODE_SYSFS:
1098 return tmc_enable_etr_sink_sysfs(csdev);
1099 case CS_MODE_PERF:
1100 return tmc_enable_etr_sink_perf(csdev);
1101 }
1102
1103 /* We shouldn't be here */
1104 return -EINVAL;
1105 }
1106
tmc_disable_etr_sink(struct coresight_device * csdev)1107 static void tmc_disable_etr_sink(struct coresight_device *csdev)
1108 {
1109 unsigned long flags;
1110 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1111
1112 spin_lock_irqsave(&drvdata->spinlock, flags);
1113 if (drvdata->reading) {
1114 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1115 return;
1116 }
1117
1118 /* Disable the TMC only if it needs to */
1119 if (drvdata->mode != CS_MODE_DISABLED) {
1120 tmc_etr_disable_hw(drvdata);
1121 drvdata->mode = CS_MODE_DISABLED;
1122 }
1123
1124 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1125
1126 dev_info(drvdata->dev, "TMC-ETR disabled\n");
1127 }
1128
1129 static const struct coresight_ops_sink tmc_etr_sink_ops = {
1130 .enable = tmc_enable_etr_sink,
1131 .disable = tmc_disable_etr_sink,
1132 };
1133
1134 const struct coresight_ops tmc_etr_cs_ops = {
1135 .sink_ops = &tmc_etr_sink_ops,
1136 };
1137
tmc_read_prepare_etr(struct tmc_drvdata * drvdata)1138 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1139 {
1140 int ret = 0;
1141 unsigned long flags;
1142
1143 /* config types are set a boot time and never change */
1144 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1145 return -EINVAL;
1146
1147 spin_lock_irqsave(&drvdata->spinlock, flags);
1148 if (drvdata->reading) {
1149 ret = -EBUSY;
1150 goto out;
1151 }
1152
1153 /* Don't interfere if operated from Perf */
1154 if (drvdata->mode == CS_MODE_PERF) {
1155 ret = -EINVAL;
1156 goto out;
1157 }
1158
1159 /* If drvdata::etr_buf is NULL the trace data has been read already */
1160 if (drvdata->etr_buf == NULL) {
1161 ret = -EINVAL;
1162 goto out;
1163 }
1164
1165 /* Disable the TMC if need be */
1166 if (drvdata->mode == CS_MODE_SYSFS)
1167 tmc_etr_disable_hw(drvdata);
1168
1169 drvdata->reading = true;
1170 out:
1171 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1172
1173 return ret;
1174 }
1175
tmc_read_unprepare_etr(struct tmc_drvdata * drvdata)1176 int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1177 {
1178 unsigned long flags;
1179 struct etr_buf *etr_buf = NULL;
1180
1181 /* config types are set a boot time and never change */
1182 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1183 return -EINVAL;
1184
1185 spin_lock_irqsave(&drvdata->spinlock, flags);
1186
1187 /* RE-enable the TMC if need be */
1188 if (drvdata->mode == CS_MODE_SYSFS) {
1189 /*
1190 * The trace run will continue with the same allocated trace
1191 * buffer. Since the tracer is still enabled drvdata::buf can't
1192 * be NULL.
1193 */
1194 tmc_etr_enable_hw(drvdata);
1195 } else {
1196 /*
1197 * The ETR is not tracing and the buffer was just read.
1198 * As such prepare to free the trace buffer.
1199 */
1200 etr_buf = drvdata->etr_buf;
1201 drvdata->etr_buf = NULL;
1202 }
1203
1204 drvdata->reading = false;
1205 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1206
1207 /* Free allocated memory out side of the spinlock */
1208 if (etr_buf)
1209 tmc_free_etr_buf(etr_buf);
1210
1211 return 0;
1212 }
1213