1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
3
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/errno.h>
7 #include <linux/interrupt.h>
8 #include <linux/pagemap.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/io.h>
11 #include <linux/platform_device.h>
12 #include <linux/uaccess.h>
13 #include <linux/mm.h>
14 #include <linux/of.h>
15 #include <soc/bcm2835/raspberrypi-firmware.h>
16
17 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
18
19 #include "vchiq_arm.h"
20 #include "vchiq_connected.h"
21 #include "vchiq_pagelist.h"
22
23 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
24
25 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
26 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1
27
28 #define BELL0 0x00
29 #define BELL2 0x08
30
31 struct vchiq_2835_state {
32 int inited;
33 struct vchiq_arm_state arm_state;
34 };
35
36 struct vchiq_pagelist_info {
37 struct pagelist *pagelist;
38 size_t pagelist_buffer_size;
39 dma_addr_t dma_addr;
40 enum dma_data_direction dma_dir;
41 unsigned int num_pages;
42 unsigned int pages_need_release;
43 struct page **pages;
44 struct scatterlist *scatterlist;
45 unsigned int scatterlist_mapped;
46 };
47
48 static void __iomem *g_regs;
49 /* This value is the size of the L2 cache lines as understood by the
50 * VPU firmware, which determines the required alignment of the
51 * offsets/sizes in pagelists.
52 *
53 * Modern VPU firmware looks for a DT "cache-line-size" property in
54 * the VCHIQ node and will overwrite it with the actual L2 cache size,
55 * which the kernel must then respect. That property was rejected
56 * upstream, so we have to use the VPU firmware's compatibility value
57 * of 32.
58 */
59 static unsigned int g_cache_line_size = 32;
60 static unsigned int g_fragments_size;
61 static char *g_fragments_base;
62 static char *g_free_fragments;
63 static struct semaphore g_free_fragments_sema;
64 static struct device *g_dev;
65
66 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
67
68 static irqreturn_t
69 vchiq_doorbell_irq(int irq, void *dev_id);
70
71 static struct vchiq_pagelist_info *
72 create_pagelist(char __user *buf, size_t count, unsigned short type);
73
74 static void
75 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
76 int actual);
77
vchiq_platform_init(struct platform_device * pdev,struct vchiq_state * state)78 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
79 {
80 struct device *dev = &pdev->dev;
81 struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
82 struct rpi_firmware *fw = drvdata->fw;
83 struct vchiq_slot_zero *vchiq_slot_zero;
84 struct resource *res;
85 void *slot_mem;
86 dma_addr_t slot_phys;
87 u32 channelbase;
88 int slot_mem_size, frag_mem_size;
89 int err, irq, i;
90
91 /*
92 * VCHI messages between the CPU and firmware use
93 * 32-bit bus addresses.
94 */
95 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
96
97 if (err < 0)
98 return err;
99
100 g_cache_line_size = drvdata->cache_line_size;
101 g_fragments_size = 2 * g_cache_line_size;
102
103 /* Allocate space for the channels in coherent memory */
104 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
105 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
106
107 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
108 &slot_phys, GFP_KERNEL);
109 if (!slot_mem) {
110 dev_err(dev, "could not allocate DMA memory\n");
111 return -ENOMEM;
112 }
113
114 WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
115
116 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
117 if (!vchiq_slot_zero)
118 return -EINVAL;
119
120 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
121 (int)slot_phys + slot_mem_size;
122 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
123 MAX_FRAGMENTS;
124
125 g_fragments_base = (char *)slot_mem + slot_mem_size;
126
127 g_free_fragments = g_fragments_base;
128 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
129 *(char **)&g_fragments_base[i*g_fragments_size] =
130 &g_fragments_base[(i + 1)*g_fragments_size];
131 }
132 *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
133 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
134
135 if (vchiq_init_state(state, vchiq_slot_zero) != VCHIQ_SUCCESS)
136 return -EINVAL;
137
138 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
139 g_regs = devm_ioremap_resource(&pdev->dev, res);
140 if (IS_ERR(g_regs))
141 return PTR_ERR(g_regs);
142
143 irq = platform_get_irq(pdev, 0);
144 if (irq <= 0)
145 return irq;
146
147 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
148 "VCHIQ doorbell", state);
149 if (err) {
150 dev_err(dev, "failed to register irq=%d\n", irq);
151 return err;
152 }
153
154 /* Send the base address of the slots to VideoCore */
155 channelbase = slot_phys;
156 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
157 &channelbase, sizeof(channelbase));
158 if (err || channelbase) {
159 dev_err(dev, "failed to set channelbase\n");
160 return err ? : -ENXIO;
161 }
162
163 g_dev = dev;
164 vchiq_log_info(vchiq_arm_log_level,
165 "vchiq_init - done (slots %pK, phys %pad)",
166 vchiq_slot_zero, &slot_phys);
167
168 vchiq_call_connected_callbacks();
169
170 return 0;
171 }
172
173 VCHIQ_STATUS_T
vchiq_platform_init_state(struct vchiq_state * state)174 vchiq_platform_init_state(struct vchiq_state *state)
175 {
176 VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
177 struct vchiq_2835_state *platform_state;
178
179 state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
180 if (!state->platform_state)
181 return VCHIQ_ERROR;
182
183 platform_state = (struct vchiq_2835_state *)state->platform_state;
184
185 platform_state->inited = 1;
186 status = vchiq_arm_init_state(state, &platform_state->arm_state);
187
188 if (status != VCHIQ_SUCCESS)
189 platform_state->inited = 0;
190
191 return status;
192 }
193
194 struct vchiq_arm_state*
vchiq_platform_get_arm_state(struct vchiq_state * state)195 vchiq_platform_get_arm_state(struct vchiq_state *state)
196 {
197 struct vchiq_2835_state *platform_state;
198
199 platform_state = (struct vchiq_2835_state *)state->platform_state;
200
201 WARN_ON_ONCE(!platform_state->inited);
202
203 return &platform_state->arm_state;
204 }
205
206 void
remote_event_signal(struct remote_event * event)207 remote_event_signal(struct remote_event *event)
208 {
209 wmb();
210
211 event->fired = 1;
212
213 dsb(sy); /* data barrier operation */
214
215 if (event->armed)
216 writel(0, g_regs + BELL2); /* trigger vc interrupt */
217 }
218
219 VCHIQ_STATUS_T
vchiq_prepare_bulk_data(struct vchiq_bulk * bulk,void * offset,int size,int dir)220 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, int size,
221 int dir)
222 {
223 struct vchiq_pagelist_info *pagelistinfo;
224
225 pagelistinfo = create_pagelist((char __user *)offset, size,
226 (dir == VCHIQ_BULK_RECEIVE)
227 ? PAGELIST_READ
228 : PAGELIST_WRITE);
229
230 if (!pagelistinfo)
231 return VCHIQ_ERROR;
232
233 bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr;
234
235 /*
236 * Store the pagelistinfo address in remote_data,
237 * which isn't used by the slave.
238 */
239 bulk->remote_data = pagelistinfo;
240
241 return VCHIQ_SUCCESS;
242 }
243
244 void
vchiq_complete_bulk(struct vchiq_bulk * bulk)245 vchiq_complete_bulk(struct vchiq_bulk *bulk)
246 {
247 if (bulk && bulk->remote_data && bulk->actual)
248 free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
249 bulk->actual);
250 }
251
252 void
vchiq_dump_platform_state(void * dump_context)253 vchiq_dump_platform_state(void *dump_context)
254 {
255 char buf[80];
256 int len;
257
258 len = snprintf(buf, sizeof(buf),
259 " Platform: 2835 (VC master)");
260 vchiq_dump(dump_context, buf, len + 1);
261 }
262
263 VCHIQ_STATUS_T
vchiq_platform_suspend(struct vchiq_state * state)264 vchiq_platform_suspend(struct vchiq_state *state)
265 {
266 return VCHIQ_ERROR;
267 }
268
269 VCHIQ_STATUS_T
vchiq_platform_resume(struct vchiq_state * state)270 vchiq_platform_resume(struct vchiq_state *state)
271 {
272 return VCHIQ_SUCCESS;
273 }
274
275 void
vchiq_platform_paused(struct vchiq_state * state)276 vchiq_platform_paused(struct vchiq_state *state)
277 {
278 }
279
280 void
vchiq_platform_resumed(struct vchiq_state * state)281 vchiq_platform_resumed(struct vchiq_state *state)
282 {
283 }
284
285 int
vchiq_platform_videocore_wanted(struct vchiq_state * state)286 vchiq_platform_videocore_wanted(struct vchiq_state *state)
287 {
288 return 1; // autosuspend not supported - videocore always wanted
289 }
290
291 int
vchiq_platform_use_suspend_timer(void)292 vchiq_platform_use_suspend_timer(void)
293 {
294 return 0;
295 }
296 void
vchiq_dump_platform_use_state(struct vchiq_state * state)297 vchiq_dump_platform_use_state(struct vchiq_state *state)
298 {
299 vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
300 }
301 void
vchiq_platform_handle_timeout(struct vchiq_state * state)302 vchiq_platform_handle_timeout(struct vchiq_state *state)
303 {
304 (void)state;
305 }
306 /*
307 * Local functions
308 */
309
310 static irqreturn_t
vchiq_doorbell_irq(int irq,void * dev_id)311 vchiq_doorbell_irq(int irq, void *dev_id)
312 {
313 struct vchiq_state *state = dev_id;
314 irqreturn_t ret = IRQ_NONE;
315 unsigned int status;
316
317 /* Read (and clear) the doorbell */
318 status = readl(g_regs + BELL0);
319
320 if (status & 0x4) { /* Was the doorbell rung? */
321 remote_event_pollall(state);
322 ret = IRQ_HANDLED;
323 }
324
325 return ret;
326 }
327
328 static void
cleanup_pagelistinfo(struct vchiq_pagelist_info * pagelistinfo)329 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
330 {
331 if (pagelistinfo->scatterlist_mapped) {
332 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
333 pagelistinfo->num_pages, pagelistinfo->dma_dir);
334 }
335
336 if (pagelistinfo->pages_need_release) {
337 unsigned int i;
338
339 for (i = 0; i < pagelistinfo->num_pages; i++)
340 put_page(pagelistinfo->pages[i]);
341 }
342
343 dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
344 pagelistinfo->pagelist, pagelistinfo->dma_addr);
345 }
346
347 /* There is a potential problem with partial cache lines (pages?)
348 * at the ends of the block when reading. If the CPU accessed anything in
349 * the same line (page?) then it may have pulled old data into the cache,
350 * obscuring the new data underneath. We can solve this by transferring the
351 * partial cache lines separately, and allowing the ARM to copy into the
352 * cached area.
353 */
354
355 static struct vchiq_pagelist_info *
create_pagelist(char __user * buf,size_t count,unsigned short type)356 create_pagelist(char __user *buf, size_t count, unsigned short type)
357 {
358 struct pagelist *pagelist;
359 struct vchiq_pagelist_info *pagelistinfo;
360 struct page **pages;
361 u32 *addrs;
362 unsigned int num_pages, offset, i, k;
363 int actual_pages;
364 size_t pagelist_size;
365 struct scatterlist *scatterlist, *sg;
366 int dma_buffers;
367 dma_addr_t dma_addr;
368
369 if (count >= INT_MAX - PAGE_SIZE)
370 return NULL;
371
372 offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1));
373 num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
374
375 if (num_pages > (SIZE_MAX - sizeof(struct pagelist) -
376 sizeof(struct vchiq_pagelist_info)) /
377 (sizeof(u32) + sizeof(pages[0]) +
378 sizeof(struct scatterlist)))
379 return NULL;
380
381 pagelist_size = sizeof(struct pagelist) +
382 (num_pages * sizeof(u32)) +
383 (num_pages * sizeof(pages[0]) +
384 (num_pages * sizeof(struct scatterlist))) +
385 sizeof(struct vchiq_pagelist_info);
386
387 /* Allocate enough storage to hold the page pointers and the page
388 * list
389 */
390 pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
391 GFP_KERNEL);
392
393 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);
394
395 if (!pagelist)
396 return NULL;
397
398 addrs = pagelist->addrs;
399 pages = (struct page **)(addrs + num_pages);
400 scatterlist = (struct scatterlist *)(pages + num_pages);
401 pagelistinfo = (struct vchiq_pagelist_info *)
402 (scatterlist + num_pages);
403
404 pagelist->length = count;
405 pagelist->type = type;
406 pagelist->offset = offset;
407
408 /* Populate the fields of the pagelistinfo structure */
409 pagelistinfo->pagelist = pagelist;
410 pagelistinfo->pagelist_buffer_size = pagelist_size;
411 pagelistinfo->dma_addr = dma_addr;
412 pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ?
413 DMA_TO_DEVICE : DMA_FROM_DEVICE;
414 pagelistinfo->num_pages = num_pages;
415 pagelistinfo->pages_need_release = 0;
416 pagelistinfo->pages = pages;
417 pagelistinfo->scatterlist = scatterlist;
418 pagelistinfo->scatterlist_mapped = 0;
419
420 if (is_vmalloc_addr(buf)) {
421 unsigned long length = count;
422 unsigned int off = offset;
423
424 for (actual_pages = 0; actual_pages < num_pages;
425 actual_pages++) {
426 struct page *pg = vmalloc_to_page(buf + (actual_pages *
427 PAGE_SIZE));
428 size_t bytes = PAGE_SIZE - off;
429
430 if (!pg) {
431 cleanup_pagelistinfo(pagelistinfo);
432 return NULL;
433 }
434
435 if (bytes > length)
436 bytes = length;
437 pages[actual_pages] = pg;
438 length -= bytes;
439 off = 0;
440 }
441 /* do not try and release vmalloc pages */
442 } else {
443 actual_pages = get_user_pages_fast(
444 (unsigned long)buf & PAGE_MASK,
445 num_pages,
446 type == PAGELIST_READ,
447 pages);
448
449 if (actual_pages != num_pages) {
450 vchiq_log_info(vchiq_arm_log_level,
451 "%s - only %d/%d pages locked",
452 __func__, actual_pages, num_pages);
453
454 /* This is probably due to the process being killed */
455 while (actual_pages > 0) {
456 actual_pages--;
457 put_page(pages[actual_pages]);
458 }
459 cleanup_pagelistinfo(pagelistinfo);
460 return NULL;
461 }
462 /* release user pages */
463 pagelistinfo->pages_need_release = 1;
464 }
465
466 /*
467 * Initialize the scatterlist so that the magic cookie
468 * is filled if debugging is enabled
469 */
470 sg_init_table(scatterlist, num_pages);
471 /* Now set the pages for each scatterlist */
472 for (i = 0; i < num_pages; i++) {
473 unsigned int len = PAGE_SIZE - offset;
474
475 if (len > count)
476 len = count;
477 sg_set_page(scatterlist + i, pages[i], len, offset);
478 offset = 0;
479 count -= len;
480 }
481
482 dma_buffers = dma_map_sg(g_dev,
483 scatterlist,
484 num_pages,
485 pagelistinfo->dma_dir);
486
487 if (dma_buffers == 0) {
488 cleanup_pagelistinfo(pagelistinfo);
489 return NULL;
490 }
491
492 pagelistinfo->scatterlist_mapped = 1;
493
494 /* Combine adjacent blocks for performance */
495 k = 0;
496 for_each_sg(scatterlist, sg, dma_buffers, i) {
497 u32 len = sg_dma_len(sg);
498 u32 addr = sg_dma_address(sg);
499
500 /* Note: addrs is the address + page_count - 1
501 * The firmware expects blocks after the first to be page-
502 * aligned and a multiple of the page size
503 */
504 WARN_ON(len == 0);
505 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
506 WARN_ON(i && (addr & ~PAGE_MASK));
507 if (k > 0 &&
508 ((addrs[k - 1] & PAGE_MASK) +
509 (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
510 == (addr & PAGE_MASK))
511 addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
512 else
513 addrs[k++] = (addr & PAGE_MASK) |
514 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
515 }
516
517 /* Partial cache lines (fragments) require special measures */
518 if ((type == PAGELIST_READ) &&
519 ((pagelist->offset & (g_cache_line_size - 1)) ||
520 ((pagelist->offset + pagelist->length) &
521 (g_cache_line_size - 1)))) {
522 char *fragments;
523
524 if (down_interruptible(&g_free_fragments_sema)) {
525 cleanup_pagelistinfo(pagelistinfo);
526 return NULL;
527 }
528
529 WARN_ON(g_free_fragments == NULL);
530
531 down(&g_free_fragments_mutex);
532 fragments = g_free_fragments;
533 WARN_ON(fragments == NULL);
534 g_free_fragments = *(char **) g_free_fragments;
535 up(&g_free_fragments_mutex);
536 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
537 (fragments - g_fragments_base) / g_fragments_size;
538 }
539
540 return pagelistinfo;
541 }
542
543 static void
free_pagelist(struct vchiq_pagelist_info * pagelistinfo,int actual)544 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
545 int actual)
546 {
547 struct pagelist *pagelist = pagelistinfo->pagelist;
548 struct page **pages = pagelistinfo->pages;
549 unsigned int num_pages = pagelistinfo->num_pages;
550
551 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
552 __func__, pagelistinfo->pagelist, actual);
553
554 /*
555 * NOTE: dma_unmap_sg must be called before the
556 * cpu can touch any of the data/pages.
557 */
558 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
559 pagelistinfo->num_pages, pagelistinfo->dma_dir);
560 pagelistinfo->scatterlist_mapped = 0;
561
562 /* Deal with any partial cache lines (fragments) */
563 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
564 char *fragments = g_fragments_base +
565 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
566 g_fragments_size;
567 int head_bytes, tail_bytes;
568
569 head_bytes = (g_cache_line_size - pagelist->offset) &
570 (g_cache_line_size - 1);
571 tail_bytes = (pagelist->offset + actual) &
572 (g_cache_line_size - 1);
573
574 if ((actual >= 0) && (head_bytes != 0)) {
575 if (head_bytes > actual)
576 head_bytes = actual;
577
578 memcpy((char *)kmap(pages[0]) +
579 pagelist->offset,
580 fragments,
581 head_bytes);
582 kunmap(pages[0]);
583 }
584 if ((actual >= 0) && (head_bytes < actual) &&
585 (tail_bytes != 0)) {
586 memcpy((char *)kmap(pages[num_pages - 1]) +
587 ((pagelist->offset + actual) &
588 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
589 fragments + g_cache_line_size,
590 tail_bytes);
591 kunmap(pages[num_pages - 1]);
592 }
593
594 down(&g_free_fragments_mutex);
595 *(char **)fragments = g_free_fragments;
596 g_free_fragments = fragments;
597 up(&g_free_fragments_mutex);
598 up(&g_free_fragments_sema);
599 }
600
601 /* Need to mark all the pages dirty. */
602 if (pagelist->type != PAGELIST_WRITE &&
603 pagelistinfo->pages_need_release) {
604 unsigned int i;
605
606 for (i = 0; i < num_pages; i++)
607 set_page_dirty(pages[i]);
608 }
609
610 cleanup_pagelistinfo(pagelistinfo);
611 }
612
