1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
3
4 #ifndef VCHIQ_CORE_H
5 #define VCHIQ_CORE_H
6
7 #include <linux/mutex.h>
8 #include <linux/completion.h>
9 #include <linux/kthread.h>
10 #include <linux/kref.h>
11 #include <linux/rcupdate.h>
12 #include <linux/wait.h>
13 #include <linux/raspberrypi/vchiq.h>
14
15 #include "vchiq_cfg.h"
16
17
18 /* Do this so that we can test-build the code on non-rpi systems */
19 #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE)
20
21 #else
22
23 #ifndef dsb
24 #define dsb(a)
25 #endif
26
27 #endif /* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */
28
29 #define VCHIQ_SERVICE_HANDLE_INVALID 0
30
31 #define VCHIQ_SLOT_SIZE 4096
32 #define VCHIQ_MAX_MSG_SIZE (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header))
33
34 /* Run time control of log level, based on KERN_XXX level. */
35 #define VCHIQ_LOG_DEFAULT 4
36 #define VCHIQ_LOG_ERROR 3
37 #define VCHIQ_LOG_WARNING 4
38 #define VCHIQ_LOG_INFO 6
39 #define VCHIQ_LOG_TRACE 7
40
41 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: "
42
43 #ifndef vchiq_log_error
44 #define vchiq_log_error(cat, fmt, ...) \
45 do { if (cat >= VCHIQ_LOG_ERROR) \
46 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
47 #endif
48 #ifndef vchiq_log_warning
49 #define vchiq_log_warning(cat, fmt, ...) \
50 do { if (cat >= VCHIQ_LOG_WARNING) \
51 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
52 #endif
53 #ifndef vchiq_log_info
54 #define vchiq_log_info(cat, fmt, ...) \
55 do { if (cat >= VCHIQ_LOG_INFO) \
56 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
57 #endif
58 #ifndef vchiq_log_trace
59 #define vchiq_log_trace(cat, fmt, ...) \
60 do { if (cat >= VCHIQ_LOG_TRACE) \
61 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
62 #endif
63
64 #define vchiq_loud_error(...) \
65 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__)
66
67 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1)
68 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
69 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \
70 VCHIQ_SLOT_SIZE)
71
72 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \
73 ((fourcc) >> 24) & 0xff, \
74 ((fourcc) >> 16) & 0xff, \
75 ((fourcc) >> 8) & 0xff, \
76 (fourcc) & 0xff
77
78 typedef uint32_t BITSET_T;
79
80 static_assert((sizeof(BITSET_T) * 8) == 32);
81
82 #define BITSET_SIZE(b) ((b + 31) >> 5)
83 #define BITSET_WORD(b) (b >> 5)
84 #define BITSET_BIT(b) (1 << (b & 31))
85 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b))
86 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
87 #define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b))
88
89 enum {
90 DEBUG_ENTRIES,
91 #if VCHIQ_ENABLE_DEBUG
92 DEBUG_SLOT_HANDLER_COUNT,
93 DEBUG_SLOT_HANDLER_LINE,
94 DEBUG_PARSE_LINE,
95 DEBUG_PARSE_HEADER,
96 DEBUG_PARSE_MSGID,
97 DEBUG_AWAIT_COMPLETION_LINE,
98 DEBUG_DEQUEUE_MESSAGE_LINE,
99 DEBUG_SERVICE_CALLBACK_LINE,
100 DEBUG_MSG_QUEUE_FULL_COUNT,
101 DEBUG_COMPLETION_QUEUE_FULL_COUNT,
102 #endif
103 DEBUG_MAX
104 };
105
106 #if VCHIQ_ENABLE_DEBUG
107
108 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug;
109 #define DEBUG_TRACE(d) \
110 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0)
111 #define DEBUG_VALUE(d, v) \
112 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0)
113 #define DEBUG_COUNT(d) \
114 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0)
115
116 #else /* VCHIQ_ENABLE_DEBUG */
117
118 #define DEBUG_INITIALISE(local)
119 #define DEBUG_TRACE(d)
120 #define DEBUG_VALUE(d, v)
121 #define DEBUG_COUNT(d)
122
123 #endif /* VCHIQ_ENABLE_DEBUG */
124
125 enum vchiq_connstate {
126 VCHIQ_CONNSTATE_DISCONNECTED,
127 VCHIQ_CONNSTATE_CONNECTING,
128 VCHIQ_CONNSTATE_CONNECTED,
129 VCHIQ_CONNSTATE_PAUSING,
130 VCHIQ_CONNSTATE_PAUSE_SENT,
131 VCHIQ_CONNSTATE_PAUSED,
132 VCHIQ_CONNSTATE_RESUMING,
133 VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
134 VCHIQ_CONNSTATE_RESUME_TIMEOUT
135 };
136
137 enum {
138 VCHIQ_SRVSTATE_FREE,
139 VCHIQ_SRVSTATE_HIDDEN,
140 VCHIQ_SRVSTATE_LISTENING,
141 VCHIQ_SRVSTATE_OPENING,
142 VCHIQ_SRVSTATE_OPEN,
143 VCHIQ_SRVSTATE_OPENSYNC,
144 VCHIQ_SRVSTATE_CLOSESENT,
145 VCHIQ_SRVSTATE_CLOSERECVD,
146 VCHIQ_SRVSTATE_CLOSEWAIT,
147 VCHIQ_SRVSTATE_CLOSED
148 };
149
150 enum vchiq_bulk_dir {
151 VCHIQ_BULK_TRANSMIT,
152 VCHIQ_BULK_RECEIVE
153 };
154
155 typedef void (*vchiq_userdata_term)(void *userdata);
156
157 struct vchiq_bulk {
158 short mode;
159 short dir;
160 void *userdata;
161 dma_addr_t data;
162 int size;
163 void *remote_data;
164 int remote_size;
165 int actual;
166 };
167
168 struct vchiq_bulk_queue {
169 int local_insert; /* Where to insert the next local bulk */
170 int remote_insert; /* Where to insert the next remote bulk (master) */
171 int process; /* Bulk to transfer next */
172 int remote_notify; /* Bulk to notify the remote client of next (mstr) */
173 int remove; /* Bulk to notify the local client of, and remove, next */
174 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS];
175 };
176
177 struct remote_event {
178 int armed;
179 int fired;
180 u32 __unused;
181 };
182
183 struct opaque_platform_state;
184
185 struct vchiq_slot {
186 char data[VCHIQ_SLOT_SIZE];
187 };
188
189 struct vchiq_slot_info {
190 /* Use two counters rather than one to avoid the need for a mutex. */
191 short use_count;
192 short release_count;
193 };
194
195 struct vchiq_service {
196 struct vchiq_service_base base;
197 unsigned int handle;
198 struct kref ref_count;
199 struct rcu_head rcu;
200 int srvstate;
201 vchiq_userdata_term userdata_term;
202 unsigned int localport;
203 unsigned int remoteport;
204 int public_fourcc;
205 int client_id;
206 char auto_close;
207 char sync;
208 char closing;
209 char trace;
210 atomic_t poll_flags;
211 short version;
212 short version_min;
213 short peer_version;
214
215 struct vchiq_state *state;
216 struct vchiq_instance *instance;
217
218 int service_use_count;
219
220 struct vchiq_bulk_queue bulk_tx;
221 struct vchiq_bulk_queue bulk_rx;
222
223 struct completion remove_event;
224 struct completion bulk_remove_event;
225 struct mutex bulk_mutex;
226
227 struct service_stats_struct {
228 int quota_stalls;
229 int slot_stalls;
230 int bulk_stalls;
231 int error_count;
232 int ctrl_tx_count;
233 int ctrl_rx_count;
234 int bulk_tx_count;
235 int bulk_rx_count;
236 int bulk_aborted_count;
237 uint64_t ctrl_tx_bytes;
238 uint64_t ctrl_rx_bytes;
239 uint64_t bulk_tx_bytes;
240 uint64_t bulk_rx_bytes;
241 } stats;
242
243 int msg_queue_read;
244 int msg_queue_write;
245 struct completion msg_queue_pop;
246 struct completion msg_queue_push;
247 struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS];
248 };
249
250 /*
251 * The quota information is outside struct vchiq_service so that it can
252 * be statically allocated, since for accounting reasons a service's slot
253 * usage is carried over between users of the same port number.
254 */
255 struct vchiq_service_quota {
256 unsigned short slot_quota;
257 unsigned short slot_use_count;
258 unsigned short message_quota;
259 unsigned short message_use_count;
260 struct completion quota_event;
261 int previous_tx_index;
262 };
263
264 struct vchiq_shared_state {
265
266 /* A non-zero value here indicates that the content is valid. */
267 int initialised;
268
269 /* The first and last (inclusive) slots allocated to the owner. */
270 int slot_first;
271 int slot_last;
272
273 /* The slot allocated to synchronous messages from the owner. */
274 int slot_sync;
275
276 /*
277 * Signalling this event indicates that owner's slot handler thread
278 * should run.
279 */
280 struct remote_event trigger;
281
282 /*
283 * Indicates the byte position within the stream where the next message
284 * will be written. The least significant bits are an index into the
285 * slot. The next bits are the index of the slot in slot_queue.
286 */
287 int tx_pos;
288
289 /* This event should be signalled when a slot is recycled. */
290 struct remote_event recycle;
291
292 /* The slot_queue index where the next recycled slot will be written. */
293 int slot_queue_recycle;
294
295 /* This event should be signalled when a synchronous message is sent. */
296 struct remote_event sync_trigger;
297
298 /*
299 * This event should be signalled when a synchronous message has been
300 * released.
301 */
302 struct remote_event sync_release;
303
304 /* A circular buffer of slot indexes. */
305 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
306
307 /* Debugging state */
308 int debug[DEBUG_MAX];
309 };
310
311 struct vchiq_slot_zero {
312 int magic;
313 short version;
314 short version_min;
315 int slot_zero_size;
316 int slot_size;
317 int max_slots;
318 int max_slots_per_side;
319 int platform_data[2];
320 struct vchiq_shared_state master;
321 struct vchiq_shared_state slave;
322 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS];
323 };
324
325 struct vchiq_state {
326 int id;
327 int initialised;
328 enum vchiq_connstate conn_state;
329 short version_common;
330
331 struct vchiq_shared_state *local;
332 struct vchiq_shared_state *remote;
333 struct vchiq_slot *slot_data;
334
335 unsigned short default_slot_quota;
336 unsigned short default_message_quota;
337
338 /* Event indicating connect message received */
339 struct completion connect;
340
341 /* Mutex protecting services */
342 struct mutex mutex;
343 struct vchiq_instance **instance;
344
345 /* Processes incoming messages */
346 struct task_struct *slot_handler_thread;
347
348 /* Processes recycled slots */
349 struct task_struct *recycle_thread;
350
351 /* Processes synchronous messages */
352 struct task_struct *sync_thread;
353
354 /* Local implementation of the trigger remote event */
355 wait_queue_head_t trigger_event;
356
357 /* Local implementation of the recycle remote event */
358 wait_queue_head_t recycle_event;
359
360 /* Local implementation of the sync trigger remote event */
361 wait_queue_head_t sync_trigger_event;
362
363 /* Local implementation of the sync release remote event */
364 wait_queue_head_t sync_release_event;
365
366 char *tx_data;
367 char *rx_data;
368 struct vchiq_slot_info *rx_info;
369
370 struct mutex slot_mutex;
371
372 struct mutex recycle_mutex;
373
374 struct mutex sync_mutex;
375
376 struct mutex bulk_transfer_mutex;
377
378 /*
379 * Indicates the byte position within the stream from where the next
380 * message will be read. The least significant bits are an index into
381 * the slot.The next bits are the index of the slot in
382 * remote->slot_queue.
383 */
384 int rx_pos;
385
386 /*
387 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read
388 * from remote->tx_pos.
389 */
390 int local_tx_pos;
391
392 /* The slot_queue index of the slot to become available next. */
393 int slot_queue_available;
394
395 /* A flag to indicate if any poll has been requested */
396 int poll_needed;
397
398 /* Ths index of the previous slot used for data messages. */
399 int previous_data_index;
400
401 /* The number of slots occupied by data messages. */
402 unsigned short data_use_count;
403
404 /* The maximum number of slots to be occupied by data messages. */
405 unsigned short data_quota;
406
407 /* An array of bit sets indicating which services must be polled. */
408 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
409
410 /* The number of the first unused service */
411 int unused_service;
412
413 /* Signalled when a free slot becomes available. */
414 struct completion slot_available_event;
415
416 struct completion slot_remove_event;
417
418 /* Signalled when a free data slot becomes available. */
419 struct completion data_quota_event;
420
421 struct state_stats_struct {
422 int slot_stalls;
423 int data_stalls;
424 int ctrl_tx_count;
425 int ctrl_rx_count;
426 int error_count;
427 } stats;
428
429 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES];
430 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES];
431 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS];
432
433 struct opaque_platform_state *platform_state;
434 };
435
436 struct bulk_waiter {
437 struct vchiq_bulk *bulk;
438 struct completion event;
439 int actual;
440 };
441
442 struct vchiq_config {
443 unsigned int max_msg_size;
444 unsigned int bulk_threshold; /* The message size above which it
445 * is better to use a bulk transfer
446 * (<= max_msg_size)
447 */
448 unsigned int max_outstanding_bulks;
449 unsigned int max_services;
450 short version; /* The version of VCHIQ */
451 short version_min; /* The minimum compatible version of VCHIQ */
452 };
453
454
455 extern spinlock_t bulk_waiter_spinlock;
456
457 extern int vchiq_core_log_level;
458 extern int vchiq_core_msg_log_level;
459 extern int vchiq_sync_log_level;
460
461 extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES];
462
463 extern const char *
464 get_conn_state_name(enum vchiq_connstate conn_state);
465
466 extern struct vchiq_slot_zero *
467 vchiq_init_slots(void *mem_base, int mem_size);
468
469 extern int
470 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero);
471
472 extern enum vchiq_status
473 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance);
474
475 struct vchiq_service *
476 vchiq_add_service_internal(struct vchiq_state *state,
477 const struct vchiq_service_params_kernel *params,
478 int srvstate, struct vchiq_instance *instance,
479 vchiq_userdata_term userdata_term);
480
481 extern enum vchiq_status
482 vchiq_open_service_internal(struct vchiq_service *service, int client_id);
483
484 extern enum vchiq_status
485 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd);
486
487 extern void
488 vchiq_terminate_service_internal(struct vchiq_service *service);
489
490 extern void
491 vchiq_free_service_internal(struct vchiq_service *service);
492
493 extern void
494 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance);
495
496 extern void
497 remote_event_pollall(struct vchiq_state *state);
498
499 extern enum vchiq_status
500 vchiq_bulk_transfer(unsigned int handle, void *offset, void __user *uoffset,
501 int size, void *userdata, enum vchiq_bulk_mode mode,
502 enum vchiq_bulk_dir dir);
503
504 extern int
505 vchiq_dump_state(void *dump_context, struct vchiq_state *state);
506
507 extern int
508 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service);
509
510 extern void
511 vchiq_loud_error_header(void);
512
513 extern void
514 vchiq_loud_error_footer(void);
515
516 extern void
517 request_poll(struct vchiq_state *state, struct vchiq_service *service,
518 int poll_type);
519
520 static inline struct vchiq_service *
handle_to_service(unsigned int handle)521 handle_to_service(unsigned int handle)
522 {
523 int idx = handle & (VCHIQ_MAX_SERVICES - 1);
524 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) &
525 (VCHIQ_MAX_STATES - 1)];
526
527 if (!state)
528 return NULL;
529 return rcu_dereference(state->services[idx]);
530 }
531
532 extern struct vchiq_service *
533 find_service_by_handle(unsigned int handle);
534
535 extern struct vchiq_service *
536 find_service_by_port(struct vchiq_state *state, int localport);
537
538 extern struct vchiq_service *
539 find_service_for_instance(struct vchiq_instance *instance,
540 unsigned int handle);
541
542 extern struct vchiq_service *
543 find_closed_service_for_instance(struct vchiq_instance *instance,
544 unsigned int handle);
545
546 extern struct vchiq_service *
547 __next_service_by_instance(struct vchiq_state *state,
548 struct vchiq_instance *instance,
549 int *pidx);
550
551 extern struct vchiq_service *
552 next_service_by_instance(struct vchiq_state *state,
553 struct vchiq_instance *instance,
554 int *pidx);
555
556 extern void
557 vchiq_service_get(struct vchiq_service *service);
558
559 extern void
560 vchiq_service_put(struct vchiq_service *service);
561
562 extern enum vchiq_status
563 vchiq_queue_message(unsigned int handle,
564 ssize_t (*copy_callback)(void *context, void *dest,
565 size_t offset, size_t maxsize),
566 void *context,
567 size_t size);
568
569 /*
570 * The following functions are called from vchiq_core, and external
571 * implementations must be provided.
572 */
573
574 extern int
575 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset,
576 void __user *uoffset, int size, int dir);
577
578 extern void
579 vchiq_complete_bulk(struct vchiq_bulk *bulk);
580
581 extern void
582 remote_event_signal(struct remote_event *event);
583
584 extern int
585 vchiq_dump(void *dump_context, const char *str, int len);
586
587 extern int
588 vchiq_dump_platform_state(void *dump_context);
589
590 extern int
591 vchiq_dump_platform_instances(void *dump_context);
592
593 extern int
594 vchiq_dump_platform_service_state(void *dump_context,
595 struct vchiq_service *service);
596
597 extern int
598 vchiq_use_service_internal(struct vchiq_service *service);
599
600 extern int
601 vchiq_release_service_internal(struct vchiq_service *service);
602
603 extern void
604 vchiq_on_remote_use(struct vchiq_state *state);
605
606 extern void
607 vchiq_on_remote_release(struct vchiq_state *state);
608
609 extern int
610 vchiq_platform_init_state(struct vchiq_state *state);
611
612 extern enum vchiq_status
613 vchiq_check_service(struct vchiq_service *service);
614
615 extern void
616 vchiq_on_remote_use_active(struct vchiq_state *state);
617
618 extern enum vchiq_status
619 vchiq_send_remote_use(struct vchiq_state *state);
620
621 extern enum vchiq_status
622 vchiq_send_remote_use_active(struct vchiq_state *state);
623
624 extern void
625 vchiq_platform_conn_state_changed(struct vchiq_state *state,
626 enum vchiq_connstate oldstate,
627 enum vchiq_connstate newstate);
628
629 extern void
630 vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate);
631
632 extern void
633 vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,
634 size_t numBytes);
635
636 extern enum vchiq_status vchiq_remove_service(unsigned int service);
637
638 extern int vchiq_get_client_id(unsigned int service);
639
640 extern void vchiq_get_config(struct vchiq_config *config);
641
642 extern int
643 vchiq_set_service_option(unsigned int service, enum vchiq_service_option option,
644 int value);
645
646 #endif
647
