1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */ 3 4 #ifndef VCHIQ_CORE_H 5 #define VCHIQ_CORE_H 6 7 #include <linux/mutex.h> 8 #include <linux/completion.h> 9 #include <linux/kthread.h> 10 #include <linux/kref.h> 11 #include <linux/rcupdate.h> 12 #include <linux/wait.h> 13 14 #include "../../include/linux/raspberrypi/vchiq.h" 15 #include "vchiq_cfg.h" 16 17 /* Do this so that we can test-build the code on non-rpi systems */ 18 #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) 19 20 #else 21 22 #ifndef dsb 23 #define dsb(a) 24 #endif 25 26 #endif /* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */ 27 28 #define VCHIQ_SERVICE_HANDLE_INVALID 0 29 30 #define VCHIQ_SLOT_SIZE 4096 31 #define VCHIQ_MAX_MSG_SIZE (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header)) 32 33 /* Run time control of log level, based on KERN_XXX level. */ 34 #define VCHIQ_LOG_DEFAULT 4 35 #define VCHIQ_LOG_ERROR 3 36 #define VCHIQ_LOG_WARNING 4 37 #define VCHIQ_LOG_INFO 6 38 #define VCHIQ_LOG_TRACE 7 39 40 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: " 41 42 #ifndef vchiq_log_error 43 #define vchiq_log_error(cat, fmt, ...) \ 44 do { if (cat >= VCHIQ_LOG_ERROR) \ 45 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 46 #endif 47 #ifndef vchiq_log_warning 48 #define vchiq_log_warning(cat, fmt, ...) \ 49 do { if (cat >= VCHIQ_LOG_WARNING) \ 50 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 51 #endif 52 #ifndef vchiq_log_info 53 #define vchiq_log_info(cat, fmt, ...) \ 54 do { if (cat >= VCHIQ_LOG_INFO) \ 55 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 56 #endif 57 #ifndef vchiq_log_trace 58 #define vchiq_log_trace(cat, fmt, ...) \ 59 do { if (cat >= VCHIQ_LOG_TRACE) \ 60 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 61 #endif 62 63 #define vchiq_loud_error(...) \ 64 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__) 65 66 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1) 67 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1) 68 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \ 69 VCHIQ_SLOT_SIZE) 70 71 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \ 72 ((fourcc) >> 24) & 0xff, \ 73 ((fourcc) >> 16) & 0xff, \ 74 ((fourcc) >> 8) & 0xff, \ 75 (fourcc) & 0xff 76 77 #define BITSET_SIZE(b) ((b + 31) >> 5) 78 #define BITSET_WORD(b) (b >> 5) 79 #define BITSET_BIT(b) (1 << (b & 31)) 80 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b)) 81 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b)) 82 83 enum { 84 DEBUG_ENTRIES, 85 #if VCHIQ_ENABLE_DEBUG 86 DEBUG_SLOT_HANDLER_COUNT, 87 DEBUG_SLOT_HANDLER_LINE, 88 DEBUG_PARSE_LINE, 89 DEBUG_PARSE_HEADER, 90 DEBUG_PARSE_MSGID, 91 DEBUG_AWAIT_COMPLETION_LINE, 92 DEBUG_DEQUEUE_MESSAGE_LINE, 93 DEBUG_SERVICE_CALLBACK_LINE, 94 DEBUG_MSG_QUEUE_FULL_COUNT, 95 DEBUG_COMPLETION_QUEUE_FULL_COUNT, 96 #endif 97 DEBUG_MAX 98 }; 99 100 #if VCHIQ_ENABLE_DEBUG 101 102 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug 103 #define DEBUG_TRACE(d) \ 104 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0) 105 #define DEBUG_VALUE(d, v) \ 106 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0) 107 #define DEBUG_COUNT(d) \ 108 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0) 109 110 #else /* VCHIQ_ENABLE_DEBUG */ 111 112 #define DEBUG_INITIALISE(local) 113 #define DEBUG_TRACE(d) 114 #define DEBUG_VALUE(d, v) 115 #define DEBUG_COUNT(d) 116 117 #endif /* VCHIQ_ENABLE_DEBUG */ 118 119 enum vchiq_connstate { 120 VCHIQ_CONNSTATE_DISCONNECTED, 121 VCHIQ_CONNSTATE_CONNECTING, 122 VCHIQ_CONNSTATE_CONNECTED, 123 VCHIQ_CONNSTATE_PAUSING, 124 VCHIQ_CONNSTATE_PAUSE_SENT, 125 VCHIQ_CONNSTATE_PAUSED, 126 VCHIQ_CONNSTATE_RESUMING, 127 VCHIQ_CONNSTATE_PAUSE_TIMEOUT, 128 VCHIQ_CONNSTATE_RESUME_TIMEOUT 129 }; 130 131 enum { 132 VCHIQ_SRVSTATE_FREE, 133 VCHIQ_SRVSTATE_HIDDEN, 134 VCHIQ_SRVSTATE_LISTENING, 135 VCHIQ_SRVSTATE_OPENING, 136 VCHIQ_SRVSTATE_OPEN, 137 VCHIQ_SRVSTATE_OPENSYNC, 138 VCHIQ_SRVSTATE_CLOSESENT, 139 VCHIQ_SRVSTATE_CLOSERECVD, 140 VCHIQ_SRVSTATE_CLOSEWAIT, 141 VCHIQ_SRVSTATE_CLOSED 142 }; 143 144 enum vchiq_bulk_dir { 145 VCHIQ_BULK_TRANSMIT, 146 VCHIQ_BULK_RECEIVE 147 }; 148 149 struct vchiq_bulk { 150 short mode; 151 short dir; 152 void *userdata; 153 dma_addr_t data; 154 int size; 155 void *remote_data; 156 int remote_size; 157 int actual; 158 }; 159 160 struct vchiq_bulk_queue { 161 int local_insert; /* Where to insert the next local bulk */ 162 int remote_insert; /* Where to insert the next remote bulk (master) */ 163 int process; /* Bulk to transfer next */ 164 int remote_notify; /* Bulk to notify the remote client of next (mstr) */ 165 int remove; /* Bulk to notify the local client of, and remove, next */ 166 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS]; 167 }; 168 169 /* 170 * Remote events provide a way of presenting several virtual doorbells to a 171 * peer (ARM host to VPU) using only one physical doorbell. They can be thought 172 * of as a way for the peer to signal a semaphore, in this case implemented as 173 * a workqueue. 174 * 175 * Remote events remain signalled until acknowledged by the receiver, and they 176 * are non-counting. They are designed in such a way as to minimise the number 177 * of interrupts and avoid unnecessary waiting. 178 * 179 * A remote_event is as small data structures that live in shared memory. It 180 * comprises two booleans - armed and fired: 181 * 182 * The sender sets fired when they signal the receiver. 183 * If fired is set, the receiver has been signalled and need not wait. 184 * The receiver sets the armed field before they begin to wait. 185 * If armed is set, the receiver is waiting and wishes to be woken by interrupt. 186 */ 187 struct remote_event { 188 int armed; 189 int fired; 190 u32 __unused; 191 }; 192 193 struct opaque_platform_state; 194 195 struct vchiq_slot { 196 char data[VCHIQ_SLOT_SIZE]; 197 }; 198 199 struct vchiq_slot_info { 200 /* Use two counters rather than one to avoid the need for a mutex. */ 201 short use_count; 202 short release_count; 203 }; 204 205 struct vchiq_service { 206 struct vchiq_service_base base; 207 unsigned int handle; 208 struct kref ref_count; 209 struct rcu_head rcu; 210 int srvstate; 211 void (*userdata_term)(void *userdata); 212 unsigned int localport; 213 unsigned int remoteport; 214 int public_fourcc; 215 int client_id; 216 char auto_close; 217 char sync; 218 char closing; 219 char trace; 220 atomic_t poll_flags; 221 short version; 222 short version_min; 223 short peer_version; 224 225 struct vchiq_state *state; 226 struct vchiq_instance *instance; 227 228 int service_use_count; 229 230 struct vchiq_bulk_queue bulk_tx; 231 struct vchiq_bulk_queue bulk_rx; 232 233 struct completion remove_event; 234 struct completion bulk_remove_event; 235 struct mutex bulk_mutex; 236 237 struct service_stats_struct { 238 int quota_stalls; 239 int slot_stalls; 240 int bulk_stalls; 241 int error_count; 242 int ctrl_tx_count; 243 int ctrl_rx_count; 244 int bulk_tx_count; 245 int bulk_rx_count; 246 int bulk_aborted_count; 247 u64 ctrl_tx_bytes; 248 u64 ctrl_rx_bytes; 249 u64 bulk_tx_bytes; 250 u64 bulk_rx_bytes; 251 } stats; 252 253 int msg_queue_read; 254 int msg_queue_write; 255 struct completion msg_queue_pop; 256 struct completion msg_queue_push; 257 struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS]; 258 }; 259 260 /* 261 * The quota information is outside struct vchiq_service so that it can 262 * be statically allocated, since for accounting reasons a service's slot 263 * usage is carried over between users of the same port number. 264 */ 265 struct vchiq_service_quota { 266 unsigned short slot_quota; 267 unsigned short slot_use_count; 268 unsigned short message_quota; 269 unsigned short message_use_count; 270 struct completion quota_event; 271 int previous_tx_index; 272 }; 273 274 struct vchiq_shared_state { 275 /* A non-zero value here indicates that the content is valid. */ 276 int initialised; 277 278 /* The first and last (inclusive) slots allocated to the owner. */ 279 int slot_first; 280 int slot_last; 281 282 /* The slot allocated to synchronous messages from the owner. */ 283 int slot_sync; 284 285 /* 286 * Signalling this event indicates that owner's slot handler thread 287 * should run. 288 */ 289 struct remote_event trigger; 290 291 /* 292 * Indicates the byte position within the stream where the next message 293 * will be written. The least significant bits are an index into the 294 * slot. The next bits are the index of the slot in slot_queue. 295 */ 296 int tx_pos; 297 298 /* This event should be signalled when a slot is recycled. */ 299 struct remote_event recycle; 300 301 /* The slot_queue index where the next recycled slot will be written. */ 302 int slot_queue_recycle; 303 304 /* This event should be signalled when a synchronous message is sent. */ 305 struct remote_event sync_trigger; 306 307 /* 308 * This event should be signalled when a synchronous message has been 309 * released. 310 */ 311 struct remote_event sync_release; 312 313 /* A circular buffer of slot indexes. */ 314 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE]; 315 316 /* Debugging state */ 317 int debug[DEBUG_MAX]; 318 }; 319 320 struct vchiq_slot_zero { 321 int magic; 322 short version; 323 short version_min; 324 int slot_zero_size; 325 int slot_size; 326 int max_slots; 327 int max_slots_per_side; 328 int platform_data[2]; 329 struct vchiq_shared_state master; 330 struct vchiq_shared_state slave; 331 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS]; 332 }; 333 334 struct vchiq_state { 335 struct device *dev; 336 int id; 337 int initialised; 338 enum vchiq_connstate conn_state; 339 short version_common; 340 341 struct vchiq_shared_state *local; 342 struct vchiq_shared_state *remote; 343 struct vchiq_slot *slot_data; 344 345 unsigned short default_slot_quota; 346 unsigned short default_message_quota; 347 348 /* Event indicating connect message received */ 349 struct completion connect; 350 351 /* Mutex protecting services */ 352 struct mutex mutex; 353 struct vchiq_instance **instance; 354 355 /* Processes incoming messages */ 356 struct task_struct *slot_handler_thread; 357 358 /* Processes recycled slots */ 359 struct task_struct *recycle_thread; 360 361 /* Processes synchronous messages */ 362 struct task_struct *sync_thread; 363 364 /* Local implementation of the trigger remote event */ 365 wait_queue_head_t trigger_event; 366 367 /* Local implementation of the recycle remote event */ 368 wait_queue_head_t recycle_event; 369 370 /* Local implementation of the sync trigger remote event */ 371 wait_queue_head_t sync_trigger_event; 372 373 /* Local implementation of the sync release remote event */ 374 wait_queue_head_t sync_release_event; 375 376 char *tx_data; 377 char *rx_data; 378 struct vchiq_slot_info *rx_info; 379 380 struct mutex slot_mutex; 381 382 struct mutex recycle_mutex; 383 384 struct mutex sync_mutex; 385 386 struct mutex bulk_transfer_mutex; 387 388 /* 389 * Indicates the byte position within the stream from where the next 390 * message will be read. The least significant bits are an index into 391 * the slot.The next bits are the index of the slot in 392 * remote->slot_queue. 393 */ 394 int rx_pos; 395 396 /* 397 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read 398 * from remote->tx_pos. 399 */ 400 int local_tx_pos; 401 402 /* The slot_queue index of the slot to become available next. */ 403 int slot_queue_available; 404 405 /* A flag to indicate if any poll has been requested */ 406 int poll_needed; 407 408 /* Ths index of the previous slot used for data messages. */ 409 int previous_data_index; 410 411 /* The number of slots occupied by data messages. */ 412 unsigned short data_use_count; 413 414 /* The maximum number of slots to be occupied by data messages. */ 415 unsigned short data_quota; 416 417 /* An array of bit sets indicating which services must be polled. */ 418 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)]; 419 420 /* The number of the first unused service */ 421 int unused_service; 422 423 /* Signalled when a free slot becomes available. */ 424 struct completion slot_available_event; 425 426 struct completion slot_remove_event; 427 428 /* Signalled when a free data slot becomes available. */ 429 struct completion data_quota_event; 430 431 struct state_stats_struct { 432 int slot_stalls; 433 int data_stalls; 434 int ctrl_tx_count; 435 int ctrl_rx_count; 436 int error_count; 437 } stats; 438 439 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES]; 440 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES]; 441 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS]; 442 443 struct opaque_platform_state *platform_state; 444 }; 445 446 struct bulk_waiter { 447 struct vchiq_bulk *bulk; 448 struct completion event; 449 int actual; 450 }; 451 452 struct vchiq_config { 453 unsigned int max_msg_size; 454 unsigned int bulk_threshold; /* The message size above which it 455 * is better to use a bulk transfer 456 * (<= max_msg_size) 457 */ 458 unsigned int max_outstanding_bulks; 459 unsigned int max_services; 460 short version; /* The version of VCHIQ */ 461 short version_min; /* The minimum compatible version of VCHIQ */ 462 }; 463 464 extern spinlock_t bulk_waiter_spinlock; 465 466 extern int vchiq_core_log_level; 467 extern int vchiq_core_msg_log_level; 468 extern int vchiq_sync_log_level; 469 470 extern const char * 471 get_conn_state_name(enum vchiq_connstate conn_state); 472 473 extern struct vchiq_slot_zero * 474 vchiq_init_slots(void *mem_base, int mem_size); 475 476 extern int 477 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero, struct device *dev); 478 479 extern int 480 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance); 481 482 struct vchiq_service * 483 vchiq_add_service_internal(struct vchiq_state *state, 484 const struct vchiq_service_params_kernel *params, 485 int srvstate, struct vchiq_instance *instance, 486 void (*userdata_term)(void *userdata)); 487 488 extern int 489 vchiq_open_service_internal(struct vchiq_service *service, int client_id); 490 491 extern int 492 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd); 493 494 extern void 495 vchiq_terminate_service_internal(struct vchiq_service *service); 496 497 extern void 498 vchiq_free_service_internal(struct vchiq_service *service); 499 500 extern void 501 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance); 502 503 extern void 504 remote_event_pollall(struct vchiq_state *state); 505 506 extern int 507 vchiq_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *offset, 508 void __user *uoffset, int size, void *userdata, enum vchiq_bulk_mode mode, 509 enum vchiq_bulk_dir dir); 510 511 extern int 512 vchiq_dump_state(void *dump_context, struct vchiq_state *state); 513 514 extern int 515 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service); 516 517 extern void 518 vchiq_loud_error_header(void); 519 520 extern void 521 vchiq_loud_error_footer(void); 522 523 extern void 524 request_poll(struct vchiq_state *state, struct vchiq_service *service, 525 int poll_type); 526 527 struct vchiq_service *handle_to_service(struct vchiq_instance *instance, unsigned int handle); 528 529 extern struct vchiq_service * 530 find_service_by_handle(struct vchiq_instance *instance, unsigned int handle); 531 532 extern struct vchiq_service * 533 find_service_by_port(struct vchiq_state *state, unsigned int localport); 534 535 extern struct vchiq_service * 536 find_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 537 538 extern struct vchiq_service * 539 find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 540 541 extern struct vchiq_service * 542 __next_service_by_instance(struct vchiq_state *state, 543 struct vchiq_instance *instance, 544 int *pidx); 545 546 extern struct vchiq_service * 547 next_service_by_instance(struct vchiq_state *state, 548 struct vchiq_instance *instance, 549 int *pidx); 550 551 extern void 552 vchiq_service_get(struct vchiq_service *service); 553 554 extern void 555 vchiq_service_put(struct vchiq_service *service); 556 557 extern int 558 vchiq_queue_message(struct vchiq_instance *instance, unsigned int handle, 559 ssize_t (*copy_callback)(void *context, void *dest, 560 size_t offset, size_t maxsize), 561 void *context, 562 size_t size); 563 564 int vchiq_prepare_bulk_data(struct vchiq_instance *instance, struct vchiq_bulk *bulk, void *offset, 565 void __user *uoffset, int size, int dir); 566 567 void vchiq_complete_bulk(struct vchiq_instance *instance, struct vchiq_bulk *bulk); 568 569 void remote_event_signal(struct remote_event *event); 570 571 int vchiq_dump(void *dump_context, const char *str, int len); 572 573 int vchiq_dump_platform_state(void *dump_context); 574 575 int vchiq_dump_platform_instances(void *dump_context); 576 577 int vchiq_dump_platform_service_state(void *dump_context, struct vchiq_service *service); 578 579 int vchiq_use_service_internal(struct vchiq_service *service); 580 581 int vchiq_release_service_internal(struct vchiq_service *service); 582 583 void vchiq_on_remote_use(struct vchiq_state *state); 584 585 void vchiq_on_remote_release(struct vchiq_state *state); 586 587 int vchiq_platform_init_state(struct vchiq_state *state); 588 589 int vchiq_check_service(struct vchiq_service *service); 590 591 void vchiq_on_remote_use_active(struct vchiq_state *state); 592 593 int vchiq_send_remote_use(struct vchiq_state *state); 594 595 int vchiq_send_remote_use_active(struct vchiq_state *state); 596 597 void vchiq_platform_conn_state_changed(struct vchiq_state *state, 598 enum vchiq_connstate oldstate, 599 enum vchiq_connstate newstate); 600 601 void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate); 602 603 void vchiq_log_dump_mem(const char *label, u32 addr, const void *void_mem, size_t num_bytes); 604 605 int vchiq_remove_service(struct vchiq_instance *instance, unsigned int service); 606 607 int vchiq_get_client_id(struct vchiq_instance *instance, unsigned int service); 608 609 void vchiq_get_config(struct vchiq_config *config); 610 611 int vchiq_set_service_option(struct vchiq_instance *instance, unsigned int service, 612 enum vchiq_service_option option, int value); 613 614 #endif 615