1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3  * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
4  * Copyright (c) 2010-2012 Broadcom. All rights reserved.
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/sched/signal.h>
10 #include <linux/types.h>
11 #include <linux/errno.h>
12 #include <linux/cdev.h>
13 #include <linux/fs.h>
14 #include <linux/device.h>
15 #include <linux/mm.h>
16 #include <linux/highmem.h>
17 #include <linux/pagemap.h>
18 #include <linux/bug.h>
19 #include <linux/completion.h>
20 #include <linux/list.h>
21 #include <linux/of.h>
22 #include <linux/platform_device.h>
23 #include <linux/compat.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/rcupdate.h>
26 #include <linux/delay.h>
27 #include <linux/slab.h>
28 #include <linux/interrupt.h>
29 #include <linux/io.h>
30 #include <linux/uaccess.h>
31 #include <soc/bcm2835/raspberrypi-firmware.h>
32 
33 #include "vchiq_core.h"
34 #include "vchiq_ioctl.h"
35 #include "vchiq_arm.h"
36 #include "vchiq_debugfs.h"
37 #include "vchiq_connected.h"
38 #include "vchiq_pagelist.h"
39 
40 #define DEVICE_NAME "vchiq"
41 
42 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
43 
44 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
45 
46 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
47 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX  1
48 
49 #define BELL0	0x00
50 #define BELL2	0x08
51 
52 #define ARM_DS_ACTIVE	BIT(2)
53 
54 /* Override the default prefix, which would be vchiq_arm (from the filename) */
55 #undef MODULE_PARAM_PREFIX
56 #define MODULE_PARAM_PREFIX DEVICE_NAME "."
57 
58 #define KEEPALIVE_VER 1
59 #define KEEPALIVE_VER_MIN KEEPALIVE_VER
60 
61 /* Run time control of log level, based on KERN_XXX level. */
62 int vchiq_arm_log_level = VCHIQ_LOG_DEFAULT;
63 int vchiq_susp_log_level = VCHIQ_LOG_ERROR;
64 
65 DEFINE_SPINLOCK(msg_queue_spinlock);
66 struct vchiq_state g_state;
67 
68 static struct platform_device *bcm2835_camera;
69 static struct platform_device *bcm2835_audio;
70 
71 struct vchiq_drvdata {
72 	const unsigned int cache_line_size;
73 	struct rpi_firmware *fw;
74 };
75 
76 static struct vchiq_drvdata bcm2835_drvdata = {
77 	.cache_line_size = 32,
78 };
79 
80 static struct vchiq_drvdata bcm2836_drvdata = {
81 	.cache_line_size = 64,
82 };
83 
84 struct vchiq_arm_state {
85 	/* Keepalive-related data */
86 	struct task_struct *ka_thread;
87 	struct completion ka_evt;
88 	atomic_t ka_use_count;
89 	atomic_t ka_use_ack_count;
90 	atomic_t ka_release_count;
91 
92 	rwlock_t susp_res_lock;
93 
94 	struct vchiq_state *state;
95 
96 	/*
97 	 * Global use count for videocore.
98 	 * This is equal to the sum of the use counts for all services.  When
99 	 * this hits zero the videocore suspend procedure will be initiated.
100 	 */
101 	int videocore_use_count;
102 
103 	/*
104 	 * Use count to track requests from videocore peer.
105 	 * This use count is not associated with a service, so needs to be
106 	 * tracked separately with the state.
107 	 */
108 	int peer_use_count;
109 
110 	/*
111 	 * Flag to indicate that the first vchiq connect has made it through.
112 	 * This means that both sides should be fully ready, and we should
113 	 * be able to suspend after this point.
114 	 */
115 	int first_connect;
116 };
117 
118 struct vchiq_2835_state {
119 	int inited;
120 	struct vchiq_arm_state arm_state;
121 };
122 
123 struct vchiq_pagelist_info {
124 	struct pagelist *pagelist;
125 	size_t pagelist_buffer_size;
126 	dma_addr_t dma_addr;
127 	enum dma_data_direction dma_dir;
128 	unsigned int num_pages;
129 	unsigned int pages_need_release;
130 	struct page **pages;
131 	struct scatterlist *scatterlist;
132 	unsigned int scatterlist_mapped;
133 };
134 
135 static void __iomem *g_regs;
136 /* This value is the size of the L2 cache lines as understood by the
137  * VPU firmware, which determines the required alignment of the
138  * offsets/sizes in pagelists.
139  *
140  * Modern VPU firmware looks for a DT "cache-line-size" property in
141  * the VCHIQ node and will overwrite it with the actual L2 cache size,
142  * which the kernel must then respect.  That property was rejected
143  * upstream, so we have to use the VPU firmware's compatibility value
144  * of 32.
145  */
146 static unsigned int g_cache_line_size = 32;
147 static unsigned int g_fragments_size;
148 static char *g_fragments_base;
149 static char *g_free_fragments;
150 static struct semaphore g_free_fragments_sema;
151 static struct device *g_dev;
152 
153 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
154 
155 static enum vchiq_status
156 vchiq_blocking_bulk_transfer(unsigned int handle, void *data,
157 			     unsigned int size, enum vchiq_bulk_dir dir);
158 
159 static irqreturn_t
vchiq_doorbell_irq(int irq,void * dev_id)160 vchiq_doorbell_irq(int irq, void *dev_id)
161 {
162 	struct vchiq_state *state = dev_id;
163 	irqreturn_t ret = IRQ_NONE;
164 	unsigned int status;
165 
166 	/* Read (and clear) the doorbell */
167 	status = readl(g_regs + BELL0);
168 
169 	if (status & ARM_DS_ACTIVE) {  /* Was the doorbell rung? */
170 		remote_event_pollall(state);
171 		ret = IRQ_HANDLED;
172 	}
173 
174 	return ret;
175 }
176 
177 static void
cleanup_pagelistinfo(struct vchiq_pagelist_info * pagelistinfo)178 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
179 {
180 	if (pagelistinfo->scatterlist_mapped) {
181 		dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
182 			     pagelistinfo->num_pages, pagelistinfo->dma_dir);
183 	}
184 
185 	if (pagelistinfo->pages_need_release)
186 		unpin_user_pages(pagelistinfo->pages, pagelistinfo->num_pages);
187 
188 	dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
189 			  pagelistinfo->pagelist, pagelistinfo->dma_addr);
190 }
191 
192 static inline bool
is_adjacent_block(u32 * addrs,u32 addr,unsigned int k)193 is_adjacent_block(u32 *addrs, u32 addr, unsigned int k)
194 {
195 	u32 tmp;
196 
197 	if (!k)
198 		return false;
199 
200 	tmp = (addrs[k - 1] & PAGE_MASK) +
201 	      (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT);
202 
203 	return tmp == (addr & PAGE_MASK);
204 }
205 
206 /* There is a potential problem with partial cache lines (pages?)
207  * at the ends of the block when reading. If the CPU accessed anything in
208  * the same line (page?) then it may have pulled old data into the cache,
209  * obscuring the new data underneath. We can solve this by transferring the
210  * partial cache lines separately, and allowing the ARM to copy into the
211  * cached area.
212  */
213 
214 static struct vchiq_pagelist_info *
create_pagelist(char * buf,char __user * ubuf,size_t count,unsigned short type)215 create_pagelist(char *buf, char __user *ubuf,
216 		size_t count, unsigned short type)
217 {
218 	struct pagelist *pagelist;
219 	struct vchiq_pagelist_info *pagelistinfo;
220 	struct page **pages;
221 	u32 *addrs;
222 	unsigned int num_pages, offset, i, k;
223 	int actual_pages;
224 	size_t pagelist_size;
225 	struct scatterlist *scatterlist, *sg;
226 	int dma_buffers;
227 	dma_addr_t dma_addr;
228 
229 	if (count >= INT_MAX - PAGE_SIZE)
230 		return NULL;
231 
232 	if (buf)
233 		offset = (uintptr_t)buf & (PAGE_SIZE - 1);
234 	else
235 		offset = (uintptr_t)ubuf & (PAGE_SIZE - 1);
236 	num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
237 
238 	if ((size_t)num_pages > (SIZE_MAX - sizeof(struct pagelist) -
239 			 sizeof(struct vchiq_pagelist_info)) /
240 			(sizeof(u32) + sizeof(pages[0]) +
241 			 sizeof(struct scatterlist)))
242 		return NULL;
243 
244 	pagelist_size = sizeof(struct pagelist) +
245 			(num_pages * sizeof(u32)) +
246 			(num_pages * sizeof(pages[0]) +
247 			(num_pages * sizeof(struct scatterlist))) +
248 			sizeof(struct vchiq_pagelist_info);
249 
250 	/* Allocate enough storage to hold the page pointers and the page
251 	 * list
252 	 */
253 	pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
254 				      GFP_KERNEL);
255 
256 	vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);
257 
258 	if (!pagelist)
259 		return NULL;
260 
261 	addrs		= pagelist->addrs;
262 	pages		= (struct page **)(addrs + num_pages);
263 	scatterlist	= (struct scatterlist *)(pages + num_pages);
264 	pagelistinfo	= (struct vchiq_pagelist_info *)
265 			  (scatterlist + num_pages);
266 
267 	pagelist->length = count;
268 	pagelist->type = type;
269 	pagelist->offset = offset;
270 
271 	/* Populate the fields of the pagelistinfo structure */
272 	pagelistinfo->pagelist = pagelist;
273 	pagelistinfo->pagelist_buffer_size = pagelist_size;
274 	pagelistinfo->dma_addr = dma_addr;
275 	pagelistinfo->dma_dir =  (type == PAGELIST_WRITE) ?
276 				  DMA_TO_DEVICE : DMA_FROM_DEVICE;
277 	pagelistinfo->num_pages = num_pages;
278 	pagelistinfo->pages_need_release = 0;
279 	pagelistinfo->pages = pages;
280 	pagelistinfo->scatterlist = scatterlist;
281 	pagelistinfo->scatterlist_mapped = 0;
282 
283 	if (buf) {
284 		unsigned long length = count;
285 		unsigned int off = offset;
286 
287 		for (actual_pages = 0; actual_pages < num_pages;
288 		     actual_pages++) {
289 			struct page *pg =
290 				vmalloc_to_page((buf +
291 						 (actual_pages * PAGE_SIZE)));
292 			size_t bytes = PAGE_SIZE - off;
293 
294 			if (!pg) {
295 				cleanup_pagelistinfo(pagelistinfo);
296 				return NULL;
297 			}
298 
299 			if (bytes > length)
300 				bytes = length;
301 			pages[actual_pages] = pg;
302 			length -= bytes;
303 			off = 0;
304 		}
305 		/* do not try and release vmalloc pages */
306 	} else {
307 		actual_pages = pin_user_pages_fast((unsigned long)ubuf & PAGE_MASK, num_pages,
308 						   type == PAGELIST_READ, pages);
309 
310 		if (actual_pages != num_pages) {
311 			vchiq_log_info(vchiq_arm_log_level,
312 				       "%s - only %d/%d pages locked",
313 				       __func__, actual_pages, num_pages);
314 
315 			/* This is probably due to the process being killed */
316 			if (actual_pages > 0)
317 				unpin_user_pages(pages, actual_pages);
318 			cleanup_pagelistinfo(pagelistinfo);
319 			return NULL;
320 		}
321 		 /* release user pages */
322 		pagelistinfo->pages_need_release = 1;
323 	}
324 
325 	/*
326 	 * Initialize the scatterlist so that the magic cookie
327 	 *  is filled if debugging is enabled
328 	 */
329 	sg_init_table(scatterlist, num_pages);
330 	/* Now set the pages for each scatterlist */
331 	for (i = 0; i < num_pages; i++)	{
332 		unsigned int len = PAGE_SIZE - offset;
333 
334 		if (len > count)
335 			len = count;
336 		sg_set_page(scatterlist + i, pages[i], len, offset);
337 		offset = 0;
338 		count -= len;
339 	}
340 
341 	dma_buffers = dma_map_sg(g_dev,
342 				 scatterlist,
343 				 num_pages,
344 				 pagelistinfo->dma_dir);
345 
346 	if (dma_buffers == 0) {
347 		cleanup_pagelistinfo(pagelistinfo);
348 		return NULL;
349 	}
350 
351 	pagelistinfo->scatterlist_mapped = 1;
352 
353 	/* Combine adjacent blocks for performance */
354 	k = 0;
355 	for_each_sg(scatterlist, sg, dma_buffers, i) {
356 		u32 len = sg_dma_len(sg);
357 		u32 addr = sg_dma_address(sg);
358 
359 		/* Note: addrs is the address + page_count - 1
360 		 * The firmware expects blocks after the first to be page-
361 		 * aligned and a multiple of the page size
362 		 */
363 		WARN_ON(len == 0);
364 		WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
365 		WARN_ON(i && (addr & ~PAGE_MASK));
366 		if (is_adjacent_block(addrs, addr, k))
367 			addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
368 		else
369 			addrs[k++] = (addr & PAGE_MASK) |
370 				(((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
371 	}
372 
373 	/* Partial cache lines (fragments) require special measures */
374 	if ((type == PAGELIST_READ) &&
375 	    ((pagelist->offset & (g_cache_line_size - 1)) ||
376 	    ((pagelist->offset + pagelist->length) &
377 	    (g_cache_line_size - 1)))) {
378 		char *fragments;
379 
380 		if (down_interruptible(&g_free_fragments_sema)) {
381 			cleanup_pagelistinfo(pagelistinfo);
382 			return NULL;
383 		}
384 
385 		WARN_ON(!g_free_fragments);
386 
387 		down(&g_free_fragments_mutex);
388 		fragments = g_free_fragments;
389 		WARN_ON(!fragments);
390 		g_free_fragments = *(char **)g_free_fragments;
391 		up(&g_free_fragments_mutex);
392 		pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
393 			(fragments - g_fragments_base) / g_fragments_size;
394 	}
395 
396 	return pagelistinfo;
397 }
398 
399 static void
free_pagelist(struct vchiq_pagelist_info * pagelistinfo,int actual)400 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
401 	      int actual)
402 {
403 	struct pagelist *pagelist = pagelistinfo->pagelist;
404 	struct page **pages = pagelistinfo->pages;
405 	unsigned int num_pages = pagelistinfo->num_pages;
406 
407 	vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
408 			__func__, pagelistinfo->pagelist, actual);
409 
410 	/*
411 	 * NOTE: dma_unmap_sg must be called before the
412 	 * cpu can touch any of the data/pages.
413 	 */
414 	dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
415 		     pagelistinfo->num_pages, pagelistinfo->dma_dir);
416 	pagelistinfo->scatterlist_mapped = 0;
417 
418 	/* Deal with any partial cache lines (fragments) */
419 	if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
420 		char *fragments = g_fragments_base +
421 			(pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
422 			g_fragments_size;
423 		int head_bytes, tail_bytes;
424 
425 		head_bytes = (g_cache_line_size - pagelist->offset) &
426 			(g_cache_line_size - 1);
427 		tail_bytes = (pagelist->offset + actual) &
428 			(g_cache_line_size - 1);
429 
430 		if ((actual >= 0) && (head_bytes != 0)) {
431 			if (head_bytes > actual)
432 				head_bytes = actual;
433 
434 			memcpy_to_page(pages[0],
435 				pagelist->offset,
436 				fragments,
437 				head_bytes);
438 		}
439 		if ((actual >= 0) && (head_bytes < actual) &&
440 		    (tail_bytes != 0))
441 			memcpy_to_page(pages[num_pages - 1],
442 				(pagelist->offset + actual) &
443 				(PAGE_SIZE - 1) & ~(g_cache_line_size - 1),
444 				fragments + g_cache_line_size,
445 				tail_bytes);
446 
447 		down(&g_free_fragments_mutex);
448 		*(char **)fragments = g_free_fragments;
449 		g_free_fragments = fragments;
450 		up(&g_free_fragments_mutex);
451 		up(&g_free_fragments_sema);
452 	}
453 
454 	/* Need to mark all the pages dirty. */
455 	if (pagelist->type != PAGELIST_WRITE &&
456 	    pagelistinfo->pages_need_release) {
457 		unsigned int i;
458 
459 		for (i = 0; i < num_pages; i++)
460 			set_page_dirty(pages[i]);
461 	}
462 
463 	cleanup_pagelistinfo(pagelistinfo);
464 }
465 
vchiq_platform_init(struct platform_device * pdev,struct vchiq_state * state)466 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
467 {
468 	struct device *dev = &pdev->dev;
469 	struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
470 	struct rpi_firmware *fw = drvdata->fw;
471 	struct vchiq_slot_zero *vchiq_slot_zero;
472 	void *slot_mem;
473 	dma_addr_t slot_phys;
474 	u32 channelbase;
475 	int slot_mem_size, frag_mem_size;
476 	int err, irq, i;
477 
478 	/*
479 	 * VCHI messages between the CPU and firmware use
480 	 * 32-bit bus addresses.
481 	 */
482 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
483 
484 	if (err < 0)
485 		return err;
486 
487 	g_cache_line_size = drvdata->cache_line_size;
488 	g_fragments_size = 2 * g_cache_line_size;
489 
490 	/* Allocate space for the channels in coherent memory */
491 	slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
492 	frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
493 
494 	slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
495 				       &slot_phys, GFP_KERNEL);
496 	if (!slot_mem) {
497 		dev_err(dev, "could not allocate DMA memory\n");
498 		return -ENOMEM;
499 	}
500 
501 	WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
502 
503 	vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
504 	if (!vchiq_slot_zero)
505 		return -EINVAL;
506 
507 	vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
508 		(int)slot_phys + slot_mem_size;
509 	vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
510 		MAX_FRAGMENTS;
511 
512 	g_fragments_base = (char *)slot_mem + slot_mem_size;
513 
514 	g_free_fragments = g_fragments_base;
515 	for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
516 		*(char **)&g_fragments_base[i * g_fragments_size] =
517 			&g_fragments_base[(i + 1) * g_fragments_size];
518 	}
519 	*(char **)&g_fragments_base[i * g_fragments_size] = NULL;
520 	sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
521 
522 	err = vchiq_init_state(state, vchiq_slot_zero);
523 	if (err)
524 		return err;
525 
526 	g_regs = devm_platform_ioremap_resource(pdev, 0);
527 	if (IS_ERR(g_regs))
528 		return PTR_ERR(g_regs);
529 
530 	irq = platform_get_irq(pdev, 0);
531 	if (irq <= 0)
532 		return irq;
533 
534 	err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
535 			       "VCHIQ doorbell", state);
536 	if (err) {
537 		dev_err(dev, "failed to register irq=%d\n", irq);
538 		return err;
539 	}
540 
541 	/* Send the base address of the slots to VideoCore */
542 	channelbase = slot_phys;
543 	err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
544 				    &channelbase, sizeof(channelbase));
545 	if (err || channelbase) {
546 		dev_err(dev, "failed to set channelbase\n");
547 		return err ? : -ENXIO;
548 	}
549 
550 	g_dev = dev;
551 	vchiq_log_info(vchiq_arm_log_level, "vchiq_init - done (slots %pK, phys %pad)",
552 		       vchiq_slot_zero, &slot_phys);
553 
554 	vchiq_call_connected_callbacks();
555 
556 	return 0;
557 }
558 
559 static void
vchiq_arm_init_state(struct vchiq_state * state,struct vchiq_arm_state * arm_state)560 vchiq_arm_init_state(struct vchiq_state *state,
561 		     struct vchiq_arm_state *arm_state)
562 {
563 	if (arm_state) {
564 		rwlock_init(&arm_state->susp_res_lock);
565 
566 		init_completion(&arm_state->ka_evt);
567 		atomic_set(&arm_state->ka_use_count, 0);
568 		atomic_set(&arm_state->ka_use_ack_count, 0);
569 		atomic_set(&arm_state->ka_release_count, 0);
570 
571 		arm_state->state = state;
572 		arm_state->first_connect = 0;
573 	}
574 }
575 
576 int
vchiq_platform_init_state(struct vchiq_state * state)577 vchiq_platform_init_state(struct vchiq_state *state)
578 {
579 	struct vchiq_2835_state *platform_state;
580 
581 	state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
582 	if (!state->platform_state)
583 		return -ENOMEM;
584 
585 	platform_state = (struct vchiq_2835_state *)state->platform_state;
586 
587 	platform_state->inited = 1;
588 	vchiq_arm_init_state(state, &platform_state->arm_state);
589 
590 	return 0;
591 }
592 
vchiq_platform_get_arm_state(struct vchiq_state * state)593 static struct vchiq_arm_state *vchiq_platform_get_arm_state(struct vchiq_state *state)
594 {
595 	struct vchiq_2835_state *platform_state;
596 
597 	platform_state   = (struct vchiq_2835_state *)state->platform_state;
598 
599 	WARN_ON_ONCE(!platform_state->inited);
600 
601 	return &platform_state->arm_state;
602 }
603 
604 void
remote_event_signal(struct remote_event * event)605 remote_event_signal(struct remote_event *event)
606 {
607 	wmb();
608 
609 	event->fired = 1;
610 
611 	dsb(sy);         /* data barrier operation */
612 
613 	if (event->armed)
614 		writel(0, g_regs + BELL2); /* trigger vc interrupt */
615 }
616 
617 int
vchiq_prepare_bulk_data(struct vchiq_bulk * bulk,void * offset,void __user * uoffset,int size,int dir)618 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset,
619 			void __user *uoffset, int size, int dir)
620 {
621 	struct vchiq_pagelist_info *pagelistinfo;
622 
623 	pagelistinfo = create_pagelist(offset, uoffset, size,
624 				       (dir == VCHIQ_BULK_RECEIVE)
625 				       ? PAGELIST_READ
626 				       : PAGELIST_WRITE);
627 
628 	if (!pagelistinfo)
629 		return -ENOMEM;
630 
631 	bulk->data = pagelistinfo->dma_addr;
632 
633 	/*
634 	 * Store the pagelistinfo address in remote_data,
635 	 * which isn't used by the slave.
636 	 */
637 	bulk->remote_data = pagelistinfo;
638 
639 	return 0;
640 }
641 
642 void
vchiq_complete_bulk(struct vchiq_bulk * bulk)643 vchiq_complete_bulk(struct vchiq_bulk *bulk)
644 {
645 	if (bulk && bulk->remote_data && bulk->actual)
646 		free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
647 			      bulk->actual);
648 }
649 
vchiq_dump_platform_state(void * dump_context)650 int vchiq_dump_platform_state(void *dump_context)
651 {
652 	char buf[80];
653 	int len;
654 
655 	len = snprintf(buf, sizeof(buf), "  Platform: 2835 (VC master)");
656 	return vchiq_dump(dump_context, buf, len + 1);
657 }
658 
659 #define VCHIQ_INIT_RETRIES 10
vchiq_initialise(struct vchiq_instance ** instance_out)660 int vchiq_initialise(struct vchiq_instance **instance_out)
661 {
662 	struct vchiq_state *state;
663 	struct vchiq_instance *instance = NULL;
664 	int i, ret;
665 
666 	/*
667 	 * VideoCore may not be ready due to boot up timing.
668 	 * It may never be ready if kernel and firmware are mismatched,so don't
669 	 * block forever.
670 	 */
671 	for (i = 0; i < VCHIQ_INIT_RETRIES; i++) {
672 		state = vchiq_get_state();
673 		if (state)
674 			break;
675 		usleep_range(500, 600);
676 	}
677 	if (i == VCHIQ_INIT_RETRIES) {
678 		vchiq_log_error(vchiq_core_log_level, "%s: videocore not initialized\n", __func__);
679 		ret = -ENOTCONN;
680 		goto failed;
681 	} else if (i > 0) {
682 		vchiq_log_warning(vchiq_core_log_level,
683 				  "%s: videocore initialized after %d retries\n", __func__, i);
684 	}
685 
686 	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
687 	if (!instance) {
688 		vchiq_log_error(vchiq_core_log_level,
689 				"%s: error allocating vchiq instance\n", __func__);
690 		ret = -ENOMEM;
691 		goto failed;
692 	}
693 
694 	instance->connected = 0;
695 	instance->state = state;
696 	mutex_init(&instance->bulk_waiter_list_mutex);
697 	INIT_LIST_HEAD(&instance->bulk_waiter_list);
698 
699 	*instance_out = instance;
700 
701 	ret = 0;
702 
703 failed:
704 	vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, ret);
705 
706 	return ret;
707 }
708 EXPORT_SYMBOL(vchiq_initialise);
709 
free_bulk_waiter(struct vchiq_instance * instance)710 void free_bulk_waiter(struct vchiq_instance *instance)
711 {
712 	struct bulk_waiter_node *waiter, *next;
713 
714 	list_for_each_entry_safe(waiter, next,
715 				 &instance->bulk_waiter_list, list) {
716 		list_del(&waiter->list);
717 		vchiq_log_info(vchiq_arm_log_level, "bulk_waiter - cleaned up %pK for pid %d",
718 			       waiter, waiter->pid);
719 		kfree(waiter);
720 	}
721 }
722 
vchiq_shutdown(struct vchiq_instance * instance)723 enum vchiq_status vchiq_shutdown(struct vchiq_instance *instance)
724 {
725 	enum vchiq_status status = VCHIQ_SUCCESS;
726 	struct vchiq_state *state = instance->state;
727 
728 	if (mutex_lock_killable(&state->mutex))
729 		return VCHIQ_RETRY;
730 
731 	/* Remove all services */
732 	vchiq_shutdown_internal(state, instance);
733 
734 	mutex_unlock(&state->mutex);
735 
736 	vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
737 
738 	free_bulk_waiter(instance);
739 	kfree(instance);
740 
741 	return status;
742 }
743 EXPORT_SYMBOL(vchiq_shutdown);
744 
vchiq_is_connected(struct vchiq_instance * instance)745 static int vchiq_is_connected(struct vchiq_instance *instance)
746 {
747 	return instance->connected;
748 }
749 
vchiq_connect(struct vchiq_instance * instance)750 enum vchiq_status vchiq_connect(struct vchiq_instance *instance)
751 {
752 	enum vchiq_status status;
753 	struct vchiq_state *state = instance->state;
754 
755 	if (mutex_lock_killable(&state->mutex)) {
756 		vchiq_log_trace(vchiq_core_log_level, "%s: call to mutex_lock failed", __func__);
757 		status = VCHIQ_RETRY;
758 		goto failed;
759 	}
760 	status = vchiq_connect_internal(state, instance);
761 
762 	if (status == VCHIQ_SUCCESS)
763 		instance->connected = 1;
764 
765 	mutex_unlock(&state->mutex);
766 
767 failed:
768 	vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
769 
770 	return status;
771 }
772 EXPORT_SYMBOL(vchiq_connect);
773 
774 static enum vchiq_status
vchiq_add_service(struct vchiq_instance * instance,const struct vchiq_service_params_kernel * params,unsigned int * phandle)775 vchiq_add_service(struct vchiq_instance *instance,
776 		  const struct vchiq_service_params_kernel *params,
777 		  unsigned int *phandle)
778 {
779 	enum vchiq_status status;
780 	struct vchiq_state *state = instance->state;
781 	struct vchiq_service *service = NULL;
782 	int srvstate;
783 
784 	*phandle = VCHIQ_SERVICE_HANDLE_INVALID;
785 
786 	srvstate = vchiq_is_connected(instance)
787 		? VCHIQ_SRVSTATE_LISTENING
788 		: VCHIQ_SRVSTATE_HIDDEN;
789 
790 	service = vchiq_add_service_internal(state, params, srvstate, instance, NULL);
791 
792 	if (service) {
793 		*phandle = service->handle;
794 		status = VCHIQ_SUCCESS;
795 	} else {
796 		status = VCHIQ_ERROR;
797 	}
798 
799 	vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
800 
801 	return status;
802 }
803 
804 enum vchiq_status
vchiq_open_service(struct vchiq_instance * instance,const struct vchiq_service_params_kernel * params,unsigned int * phandle)805 vchiq_open_service(struct vchiq_instance *instance,
806 		   const struct vchiq_service_params_kernel *params,
807 		   unsigned int *phandle)
808 {
809 	enum vchiq_status   status = VCHIQ_ERROR;
810 	struct vchiq_state   *state = instance->state;
811 	struct vchiq_service *service = NULL;
812 
813 	*phandle = VCHIQ_SERVICE_HANDLE_INVALID;
814 
815 	if (!vchiq_is_connected(instance))
816 		goto failed;
817 
818 	service = vchiq_add_service_internal(state, params, VCHIQ_SRVSTATE_OPENING, instance, NULL);
819 
820 	if (service) {
821 		*phandle = service->handle;
822 		status = vchiq_open_service_internal(service, current->pid);
823 		if (status != VCHIQ_SUCCESS) {
824 			vchiq_remove_service(service->handle);
825 			*phandle = VCHIQ_SERVICE_HANDLE_INVALID;
826 		}
827 	}
828 
829 failed:
830 	vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status);
831 
832 	return status;
833 }
834 EXPORT_SYMBOL(vchiq_open_service);
835 
836 enum vchiq_status
vchiq_bulk_transmit(unsigned int handle,const void * data,unsigned int size,void * userdata,enum vchiq_bulk_mode mode)837 vchiq_bulk_transmit(unsigned int handle, const void *data, unsigned int size,
838 		    void *userdata, enum vchiq_bulk_mode mode)
839 {
840 	enum vchiq_status status;
841 
842 	while (1) {
843 		switch (mode) {
844 		case VCHIQ_BULK_MODE_NOCALLBACK:
845 		case VCHIQ_BULK_MODE_CALLBACK:
846 			status = vchiq_bulk_transfer(handle,
847 						     (void *)data, NULL,
848 						     size, userdata, mode,
849 						     VCHIQ_BULK_TRANSMIT);
850 			break;
851 		case VCHIQ_BULK_MODE_BLOCKING:
852 			status = vchiq_blocking_bulk_transfer(handle, (void *)data, size,
853 							      VCHIQ_BULK_TRANSMIT);
854 			break;
855 		default:
856 			return VCHIQ_ERROR;
857 		}
858 
859 		/*
860 		 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need
861 		 * to implement a retry mechanism since this function is
862 		 * supposed to block until queued
863 		 */
864 		if (status != VCHIQ_RETRY)
865 			break;
866 
867 		msleep(1);
868 	}
869 
870 	return status;
871 }
872 EXPORT_SYMBOL(vchiq_bulk_transmit);
873 
vchiq_bulk_receive(unsigned int handle,void * data,unsigned int size,void * userdata,enum vchiq_bulk_mode mode)874 enum vchiq_status vchiq_bulk_receive(unsigned int handle, void *data,
875 				     unsigned int size, void *userdata,
876 				     enum vchiq_bulk_mode mode)
877 {
878 	enum vchiq_status status;
879 
880 	while (1) {
881 		switch (mode) {
882 		case VCHIQ_BULK_MODE_NOCALLBACK:
883 		case VCHIQ_BULK_MODE_CALLBACK:
884 			status = vchiq_bulk_transfer(handle, data, NULL,
885 						     size, userdata,
886 						     mode, VCHIQ_BULK_RECEIVE);
887 			break;
888 		case VCHIQ_BULK_MODE_BLOCKING:
889 			status = vchiq_blocking_bulk_transfer(handle, (void *)data, size,
890 							      VCHIQ_BULK_RECEIVE);
891 			break;
892 		default:
893 			return VCHIQ_ERROR;
894 		}
895 
896 		/*
897 		 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need
898 		 * to implement a retry mechanism since this function is
899 		 * supposed to block until queued
900 		 */
901 		if (status != VCHIQ_RETRY)
902 			break;
903 
904 		msleep(1);
905 	}
906 
907 	return status;
908 }
909 EXPORT_SYMBOL(vchiq_bulk_receive);
910 
911 static enum vchiq_status
vchiq_blocking_bulk_transfer(unsigned int handle,void * data,unsigned int size,enum vchiq_bulk_dir dir)912 vchiq_blocking_bulk_transfer(unsigned int handle, void *data, unsigned int size,
913 			     enum vchiq_bulk_dir dir)
914 {
915 	struct vchiq_instance *instance;
916 	struct vchiq_service *service;
917 	enum vchiq_status status;
918 	struct bulk_waiter_node *waiter = NULL, *iter;
919 
920 	service = find_service_by_handle(handle);
921 	if (!service)
922 		return VCHIQ_ERROR;
923 
924 	instance = service->instance;
925 
926 	vchiq_service_put(service);
927 
928 	mutex_lock(&instance->bulk_waiter_list_mutex);
929 	list_for_each_entry(iter, &instance->bulk_waiter_list, list) {
930 		if (iter->pid == current->pid) {
931 			list_del(&iter->list);
932 			waiter = iter;
933 			break;
934 		}
935 	}
936 	mutex_unlock(&instance->bulk_waiter_list_mutex);
937 
938 	if (waiter) {
939 		struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk;
940 
941 		if (bulk) {
942 			/* This thread has an outstanding bulk transfer. */
943 			/* FIXME: why compare a dma address to a pointer? */
944 			if ((bulk->data != (dma_addr_t)(uintptr_t)data) || (bulk->size != size)) {
945 				/*
946 				 * This is not a retry of the previous one.
947 				 * Cancel the signal when the transfer completes.
948 				 */
949 				spin_lock(&bulk_waiter_spinlock);
950 				bulk->userdata = NULL;
951 				spin_unlock(&bulk_waiter_spinlock);
952 			}
953 		}
954 	} else {
955 		waiter = kzalloc(sizeof(*waiter), GFP_KERNEL);
956 		if (!waiter) {
957 			vchiq_log_error(vchiq_core_log_level, "%s - out of memory", __func__);
958 			return VCHIQ_ERROR;
959 		}
960 	}
961 
962 	status = vchiq_bulk_transfer(handle, data, NULL, size,
963 				     &waiter->bulk_waiter,
964 				     VCHIQ_BULK_MODE_BLOCKING, dir);
965 	if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) || !waiter->bulk_waiter.bulk) {
966 		struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk;
967 
968 		if (bulk) {
969 			/* Cancel the signal when the transfer completes. */
970 			spin_lock(&bulk_waiter_spinlock);
971 			bulk->userdata = NULL;
972 			spin_unlock(&bulk_waiter_spinlock);
973 		}
974 		kfree(waiter);
975 	} else {
976 		waiter->pid = current->pid;
977 		mutex_lock(&instance->bulk_waiter_list_mutex);
978 		list_add(&waiter->list, &instance->bulk_waiter_list);
979 		mutex_unlock(&instance->bulk_waiter_list_mutex);
980 		vchiq_log_info(vchiq_arm_log_level, "saved bulk_waiter %pK for pid %d", waiter,
981 			       current->pid);
982 	}
983 
984 	return status;
985 }
986 
987 static enum vchiq_status
add_completion(struct vchiq_instance * instance,enum vchiq_reason reason,struct vchiq_header * header,struct user_service * user_service,void * bulk_userdata)988 add_completion(struct vchiq_instance *instance, enum vchiq_reason reason,
989 	       struct vchiq_header *header, struct user_service *user_service,
990 	       void *bulk_userdata)
991 {
992 	struct vchiq_completion_data_kernel *completion;
993 	int insert;
994 
995 	DEBUG_INITIALISE(g_state.local);
996 
997 	insert = instance->completion_insert;
998 	while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) {
999 		/* Out of space - wait for the client */
1000 		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1001 		vchiq_log_trace(vchiq_arm_log_level, "%s - completion queue full", __func__);
1002 		DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);
1003 		if (wait_for_completion_interruptible(&instance->remove_event)) {
1004 			vchiq_log_info(vchiq_arm_log_level, "service_callback interrupted");
1005 			return VCHIQ_RETRY;
1006 		} else if (instance->closing) {
1007 			vchiq_log_info(vchiq_arm_log_level, "service_callback closing");
1008 			return VCHIQ_SUCCESS;
1009 		}
1010 		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1011 	}
1012 
1013 	completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)];
1014 
1015 	completion->header = header;
1016 	completion->reason = reason;
1017 	/* N.B. service_userdata is updated while processing AWAIT_COMPLETION */
1018 	completion->service_userdata = user_service->service;
1019 	completion->bulk_userdata = bulk_userdata;
1020 
1021 	if (reason == VCHIQ_SERVICE_CLOSED) {
1022 		/*
1023 		 * Take an extra reference, to be held until
1024 		 * this CLOSED notification is delivered.
1025 		 */
1026 		vchiq_service_get(user_service->service);
1027 		if (instance->use_close_delivered)
1028 			user_service->close_pending = 1;
1029 	}
1030 
1031 	/*
1032 	 * A write barrier is needed here to ensure that the entire completion
1033 	 * record is written out before the insert point.
1034 	 */
1035 	wmb();
1036 
1037 	if (reason == VCHIQ_MESSAGE_AVAILABLE)
1038 		user_service->message_available_pos = insert;
1039 
1040 	insert++;
1041 	instance->completion_insert = insert;
1042 
1043 	complete(&instance->insert_event);
1044 
1045 	return VCHIQ_SUCCESS;
1046 }
1047 
1048 enum vchiq_status
service_callback(enum vchiq_reason reason,struct vchiq_header * header,unsigned int handle,void * bulk_userdata)1049 service_callback(enum vchiq_reason reason, struct vchiq_header *header,
1050 		 unsigned int handle, void *bulk_userdata)
1051 {
1052 	/*
1053 	 * How do we ensure the callback goes to the right client?
1054 	 * The service_user data points to a user_service record
1055 	 * containing the original callback and the user state structure, which
1056 	 * contains a circular buffer for completion records.
1057 	 */
1058 	struct user_service *user_service;
1059 	struct vchiq_service *service;
1060 	struct vchiq_instance *instance;
1061 	bool skip_completion = false;
1062 
1063 	DEBUG_INITIALISE(g_state.local);
1064 
1065 	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1066 
1067 	rcu_read_lock();
1068 	service = handle_to_service(handle);
1069 	if (WARN_ON(!service)) {
1070 		rcu_read_unlock();
1071 		return VCHIQ_SUCCESS;
1072 	}
1073 
1074 	user_service = (struct user_service *)service->base.userdata;
1075 	instance = user_service->instance;
1076 
1077 	if (!instance || instance->closing) {
1078 		rcu_read_unlock();
1079 		return VCHIQ_SUCCESS;
1080 	}
1081 
1082 	/*
1083 	 * As hopping around different synchronization mechanism,
1084 	 * taking an extra reference results in simpler implementation.
1085 	 */
1086 	vchiq_service_get(service);
1087 	rcu_read_unlock();
1088 
1089 	vchiq_log_trace(vchiq_arm_log_level,
1090 			"%s - service %lx(%d,%p), reason %d, header %lx, instance %lx, bulk_userdata %lx",
1091 			__func__, (unsigned long)user_service, service->localport,
1092 			user_service->userdata, reason, (unsigned long)header,
1093 			(unsigned long)instance, (unsigned long)bulk_userdata);
1094 
1095 	if (header && user_service->is_vchi) {
1096 		spin_lock(&msg_queue_spinlock);
1097 		while (user_service->msg_insert ==
1098 			(user_service->msg_remove + MSG_QUEUE_SIZE)) {
1099 			spin_unlock(&msg_queue_spinlock);
1100 			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1101 			DEBUG_COUNT(MSG_QUEUE_FULL_COUNT);
1102 			vchiq_log_trace(vchiq_arm_log_level, "%s - msg queue full", __func__);
1103 			/*
1104 			 * If there is no MESSAGE_AVAILABLE in the completion
1105 			 * queue, add one
1106 			 */
1107 			if ((user_service->message_available_pos -
1108 				instance->completion_remove) < 0) {
1109 				enum vchiq_status status;
1110 
1111 				vchiq_log_info(vchiq_arm_log_level,
1112 					       "Inserting extra MESSAGE_AVAILABLE");
1113 				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1114 				status = add_completion(instance, reason, NULL, user_service,
1115 							bulk_userdata);
1116 				if (status != VCHIQ_SUCCESS) {
1117 					DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1118 					vchiq_service_put(service);
1119 					return status;
1120 				}
1121 			}
1122 
1123 			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1124 			if (wait_for_completion_interruptible(&user_service->remove_event)) {
1125 				vchiq_log_info(vchiq_arm_log_level, "%s interrupted", __func__);
1126 				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1127 				vchiq_service_put(service);
1128 				return VCHIQ_RETRY;
1129 			} else if (instance->closing) {
1130 				vchiq_log_info(vchiq_arm_log_level, "%s closing", __func__);
1131 				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1132 				vchiq_service_put(service);
1133 				return VCHIQ_ERROR;
1134 			}
1135 			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1136 			spin_lock(&msg_queue_spinlock);
1137 		}
1138 
1139 		user_service->msg_queue[user_service->msg_insert &
1140 			(MSG_QUEUE_SIZE - 1)] = header;
1141 		user_service->msg_insert++;
1142 
1143 		/*
1144 		 * If there is a thread waiting in DEQUEUE_MESSAGE, or if
1145 		 * there is a MESSAGE_AVAILABLE in the completion queue then
1146 		 * bypass the completion queue.
1147 		 */
1148 		if (((user_service->message_available_pos -
1149 			instance->completion_remove) >= 0) ||
1150 			user_service->dequeue_pending) {
1151 			user_service->dequeue_pending = 0;
1152 			skip_completion = true;
1153 		}
1154 
1155 		spin_unlock(&msg_queue_spinlock);
1156 		complete(&user_service->insert_event);
1157 
1158 		header = NULL;
1159 	}
1160 	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
1161 	vchiq_service_put(service);
1162 
1163 	if (skip_completion)
1164 		return VCHIQ_SUCCESS;
1165 
1166 	return add_completion(instance, reason, header, user_service,
1167 		bulk_userdata);
1168 }
1169 
vchiq_dump(void * dump_context,const char * str,int len)1170 int vchiq_dump(void *dump_context, const char *str, int len)
1171 {
1172 	struct dump_context *context = (struct dump_context *)dump_context;
1173 	int copy_bytes;
1174 
1175 	if (context->actual >= context->space)
1176 		return 0;
1177 
1178 	if (context->offset > 0) {
1179 		int skip_bytes = min_t(int, len, context->offset);
1180 
1181 		str += skip_bytes;
1182 		len -= skip_bytes;
1183 		context->offset -= skip_bytes;
1184 		if (context->offset > 0)
1185 			return 0;
1186 	}
1187 	copy_bytes = min_t(int, len, context->space - context->actual);
1188 	if (copy_bytes == 0)
1189 		return 0;
1190 	if (copy_to_user(context->buf + context->actual, str,
1191 			 copy_bytes))
1192 		return -EFAULT;
1193 	context->actual += copy_bytes;
1194 	len -= copy_bytes;
1195 
1196 	/*
1197 	 * If the terminating NUL is included in the length, then it
1198 	 * marks the end of a line and should be replaced with a
1199 	 * carriage return.
1200 	 */
1201 	if ((len == 0) && (str[copy_bytes - 1] == '\0')) {
1202 		char cr = '\n';
1203 
1204 		if (copy_to_user(context->buf + context->actual - 1,
1205 				 &cr, 1))
1206 			return -EFAULT;
1207 	}
1208 	return 0;
1209 }
1210 
vchiq_dump_platform_instances(void * dump_context)1211 int vchiq_dump_platform_instances(void *dump_context)
1212 {
1213 	struct vchiq_state *state = vchiq_get_state();
1214 	char buf[80];
1215 	int len;
1216 	int i;
1217 
1218 	if (!state)
1219 		return -ENOTCONN;
1220 
1221 	/*
1222 	 * There is no list of instances, so instead scan all services,
1223 	 * marking those that have been dumped.
1224 	 */
1225 
1226 	rcu_read_lock();
1227 	for (i = 0; i < state->unused_service; i++) {
1228 		struct vchiq_service *service;
1229 		struct vchiq_instance *instance;
1230 
1231 		service = rcu_dereference(state->services[i]);
1232 		if (!service || service->base.callback != service_callback)
1233 			continue;
1234 
1235 		instance = service->instance;
1236 		if (instance)
1237 			instance->mark = 0;
1238 	}
1239 	rcu_read_unlock();
1240 
1241 	for (i = 0; i < state->unused_service; i++) {
1242 		struct vchiq_service *service;
1243 		struct vchiq_instance *instance;
1244 		int err;
1245 
1246 		rcu_read_lock();
1247 		service = rcu_dereference(state->services[i]);
1248 		if (!service || service->base.callback != service_callback) {
1249 			rcu_read_unlock();
1250 			continue;
1251 		}
1252 
1253 		instance = service->instance;
1254 		if (!instance || instance->mark) {
1255 			rcu_read_unlock();
1256 			continue;
1257 		}
1258 		rcu_read_unlock();
1259 
1260 		len = snprintf(buf, sizeof(buf),
1261 			       "Instance %pK: pid %d,%s completions %d/%d",
1262 			       instance, instance->pid,
1263 			       instance->connected ? " connected, " :
1264 			       "",
1265 			       instance->completion_insert -
1266 			       instance->completion_remove,
1267 			       MAX_COMPLETIONS);
1268 		err = vchiq_dump(dump_context, buf, len + 1);
1269 		if (err)
1270 			return err;
1271 		instance->mark = 1;
1272 	}
1273 	return 0;
1274 }
1275 
vchiq_dump_platform_service_state(void * dump_context,struct vchiq_service * service)1276 int vchiq_dump_platform_service_state(void *dump_context,
1277 				      struct vchiq_service *service)
1278 {
1279 	struct user_service *user_service =
1280 			(struct user_service *)service->base.userdata;
1281 	char buf[80];
1282 	int len;
1283 
1284 	len = scnprintf(buf, sizeof(buf), "  instance %pK", service->instance);
1285 
1286 	if ((service->base.callback == service_callback) && user_service->is_vchi) {
1287 		len += scnprintf(buf + len, sizeof(buf) - len, ", %d/%d messages",
1288 				 user_service->msg_insert - user_service->msg_remove,
1289 				 MSG_QUEUE_SIZE);
1290 
1291 		if (user_service->dequeue_pending)
1292 			len += scnprintf(buf + len, sizeof(buf) - len,
1293 				" (dequeue pending)");
1294 	}
1295 
1296 	return vchiq_dump(dump_context, buf, len + 1);
1297 }
1298 
1299 struct vchiq_state *
vchiq_get_state(void)1300 vchiq_get_state(void)
1301 {
1302 	if (!g_state.remote) {
1303 		pr_err("%s: g_state.remote == NULL\n", __func__);
1304 		return NULL;
1305 	}
1306 
1307 	if (g_state.remote->initialised != 1) {
1308 		pr_notice("%s: g_state.remote->initialised != 1 (%d)\n",
1309 			  __func__, g_state.remote->initialised);
1310 		return NULL;
1311 	}
1312 
1313 	return &g_state;
1314 }
1315 
1316 /*
1317  * Autosuspend related functionality
1318  */
1319 
1320 static enum vchiq_status
vchiq_keepalive_vchiq_callback(enum vchiq_reason reason,struct vchiq_header * header,unsigned int service_user,void * bulk_user)1321 vchiq_keepalive_vchiq_callback(enum vchiq_reason reason,
1322 			       struct vchiq_header *header,
1323 			       unsigned int service_user, void *bulk_user)
1324 {
1325 	vchiq_log_error(vchiq_susp_log_level, "%s callback reason %d", __func__, reason);
1326 	return 0;
1327 }
1328 
1329 static int
vchiq_keepalive_thread_func(void * v)1330 vchiq_keepalive_thread_func(void *v)
1331 {
1332 	struct vchiq_state *state = (struct vchiq_state *)v;
1333 	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1334 
1335 	enum vchiq_status status;
1336 	struct vchiq_instance *instance;
1337 	unsigned int ka_handle;
1338 	int ret;
1339 
1340 	struct vchiq_service_params_kernel params = {
1341 		.fourcc      = VCHIQ_MAKE_FOURCC('K', 'E', 'E', 'P'),
1342 		.callback    = vchiq_keepalive_vchiq_callback,
1343 		.version     = KEEPALIVE_VER,
1344 		.version_min = KEEPALIVE_VER_MIN
1345 	};
1346 
1347 	ret = vchiq_initialise(&instance);
1348 	if (ret) {
1349 		vchiq_log_error(vchiq_susp_log_level, "%s vchiq_initialise failed %d", __func__,
1350 				ret);
1351 		goto exit;
1352 	}
1353 
1354 	status = vchiq_connect(instance);
1355 	if (status != VCHIQ_SUCCESS) {
1356 		vchiq_log_error(vchiq_susp_log_level, "%s vchiq_connect failed %d", __func__,
1357 				status);
1358 		goto shutdown;
1359 	}
1360 
1361 	status = vchiq_add_service(instance, &params, &ka_handle);
1362 	if (status != VCHIQ_SUCCESS) {
1363 		vchiq_log_error(vchiq_susp_log_level, "%s vchiq_open_service failed %d", __func__,
1364 				status);
1365 		goto shutdown;
1366 	}
1367 
1368 	while (1) {
1369 		long rc = 0, uc = 0;
1370 
1371 		if (wait_for_completion_interruptible(&arm_state->ka_evt)) {
1372 			vchiq_log_error(vchiq_susp_log_level, "%s interrupted", __func__);
1373 			flush_signals(current);
1374 			continue;
1375 		}
1376 
1377 		/*
1378 		 * read and clear counters.  Do release_count then use_count to
1379 		 * prevent getting more releases than uses
1380 		 */
1381 		rc = atomic_xchg(&arm_state->ka_release_count, 0);
1382 		uc = atomic_xchg(&arm_state->ka_use_count, 0);
1383 
1384 		/*
1385 		 * Call use/release service the requisite number of times.
1386 		 * Process use before release so use counts don't go negative
1387 		 */
1388 		while (uc--) {
1389 			atomic_inc(&arm_state->ka_use_ack_count);
1390 			status = vchiq_use_service(ka_handle);
1391 			if (status != VCHIQ_SUCCESS) {
1392 				vchiq_log_error(vchiq_susp_log_level,
1393 						"%s vchiq_use_service error %d", __func__, status);
1394 			}
1395 		}
1396 		while (rc--) {
1397 			status = vchiq_release_service(ka_handle);
1398 			if (status != VCHIQ_SUCCESS) {
1399 				vchiq_log_error(vchiq_susp_log_level,
1400 						"%s vchiq_release_service error %d", __func__,
1401 						status);
1402 			}
1403 		}
1404 	}
1405 
1406 shutdown:
1407 	vchiq_shutdown(instance);
1408 exit:
1409 	return 0;
1410 }
1411 
1412 int
vchiq_use_internal(struct vchiq_state * state,struct vchiq_service * service,enum USE_TYPE_E use_type)1413 vchiq_use_internal(struct vchiq_state *state, struct vchiq_service *service,
1414 		   enum USE_TYPE_E use_type)
1415 {
1416 	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1417 	int ret = 0;
1418 	char entity[16];
1419 	int *entity_uc;
1420 	int local_uc;
1421 
1422 	if (!arm_state) {
1423 		ret = -EINVAL;
1424 		goto out;
1425 	}
1426 
1427 	if (use_type == USE_TYPE_VCHIQ) {
1428 		sprintf(entity, "VCHIQ:   ");
1429 		entity_uc = &arm_state->peer_use_count;
1430 	} else if (service) {
1431 		sprintf(entity, "%c%c%c%c:%03d",
1432 			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
1433 			service->client_id);
1434 		entity_uc = &service->service_use_count;
1435 	} else {
1436 		vchiq_log_error(vchiq_susp_log_level, "%s null service ptr", __func__);
1437 		ret = -EINVAL;
1438 		goto out;
1439 	}
1440 
1441 	write_lock_bh(&arm_state->susp_res_lock);
1442 	local_uc = ++arm_state->videocore_use_count;
1443 	++(*entity_uc);
1444 
1445 	vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity,
1446 			*entity_uc, local_uc);
1447 
1448 	write_unlock_bh(&arm_state->susp_res_lock);
1449 
1450 	if (!ret) {
1451 		enum vchiq_status status = VCHIQ_SUCCESS;
1452 		long ack_cnt = atomic_xchg(&arm_state->ka_use_ack_count, 0);
1453 
1454 		while (ack_cnt && (status == VCHIQ_SUCCESS)) {
1455 			/* Send the use notify to videocore */
1456 			status = vchiq_send_remote_use_active(state);
1457 			if (status == VCHIQ_SUCCESS)
1458 				ack_cnt--;
1459 			else
1460 				atomic_add(ack_cnt, &arm_state->ka_use_ack_count);
1461 		}
1462 	}
1463 
1464 out:
1465 	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
1466 	return ret;
1467 }
1468 
1469 int
vchiq_release_internal(struct vchiq_state * state,struct vchiq_service * service)1470 vchiq_release_internal(struct vchiq_state *state, struct vchiq_service *service)
1471 {
1472 	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1473 	int ret = 0;
1474 	char entity[16];
1475 	int *entity_uc;
1476 
1477 	if (!arm_state) {
1478 		ret = -EINVAL;
1479 		goto out;
1480 	}
1481 
1482 	if (service) {
1483 		sprintf(entity, "%c%c%c%c:%03d",
1484 			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
1485 			service->client_id);
1486 		entity_uc = &service->service_use_count;
1487 	} else {
1488 		sprintf(entity, "PEER:   ");
1489 		entity_uc = &arm_state->peer_use_count;
1490 	}
1491 
1492 	write_lock_bh(&arm_state->susp_res_lock);
1493 	if (!arm_state->videocore_use_count || !(*entity_uc)) {
1494 		/* Don't use BUG_ON - don't allow user thread to crash kernel */
1495 		WARN_ON(!arm_state->videocore_use_count);
1496 		WARN_ON(!(*entity_uc));
1497 		ret = -EINVAL;
1498 		goto unlock;
1499 	}
1500 	--arm_state->videocore_use_count;
1501 	--(*entity_uc);
1502 
1503 	vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity,
1504 			*entity_uc, arm_state->videocore_use_count);
1505 
1506 unlock:
1507 	write_unlock_bh(&arm_state->susp_res_lock);
1508 
1509 out:
1510 	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
1511 	return ret;
1512 }
1513 
1514 void
vchiq_on_remote_use(struct vchiq_state * state)1515 vchiq_on_remote_use(struct vchiq_state *state)
1516 {
1517 	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1518 
1519 	atomic_inc(&arm_state->ka_use_count);
1520 	complete(&arm_state->ka_evt);
1521 }
1522 
1523 void
vchiq_on_remote_release(struct vchiq_state * state)1524 vchiq_on_remote_release(struct vchiq_state *state)
1525 {
1526 	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1527 
1528 	atomic_inc(&arm_state->ka_release_count);
1529 	complete(&arm_state->ka_evt);
1530 }
1531 
1532 int
vchiq_use_service_internal(struct vchiq_service * service)1533 vchiq_use_service_internal(struct vchiq_service *service)
1534 {
1535 	return vchiq_use_internal(service->state, service, USE_TYPE_SERVICE);
1536 }
1537 
1538 int
vchiq_release_service_internal(struct vchiq_service * service)1539 vchiq_release_service_internal(struct vchiq_service *service)
1540 {
1541 	return vchiq_release_internal(service->state, service);
1542 }
1543 
1544 struct vchiq_debugfs_node *
vchiq_instance_get_debugfs_node(struct vchiq_instance * instance)1545 vchiq_instance_get_debugfs_node(struct vchiq_instance *instance)
1546 {
1547 	return &instance->debugfs_node;
1548 }
1549 
1550 int
vchiq_instance_get_use_count(struct vchiq_instance * instance)1551 vchiq_instance_get_use_count(struct vchiq_instance *instance)
1552 {
1553 	struct vchiq_service *service;
1554 	int use_count = 0, i;
1555 
1556 	i = 0;
1557 	rcu_read_lock();
1558 	while ((service = __next_service_by_instance(instance->state,
1559 						     instance, &i)))
1560 		use_count += service->service_use_count;
1561 	rcu_read_unlock();
1562 	return use_count;
1563 }
1564 
1565 int
vchiq_instance_get_pid(struct vchiq_instance * instance)1566 vchiq_instance_get_pid(struct vchiq_instance *instance)
1567 {
1568 	return instance->pid;
1569 }
1570 
1571 int
vchiq_instance_get_trace(struct vchiq_instance * instance)1572 vchiq_instance_get_trace(struct vchiq_instance *instance)
1573 {
1574 	return instance->trace;
1575 }
1576 
1577 void
vchiq_instance_set_trace(struct vchiq_instance * instance,int trace)1578 vchiq_instance_set_trace(struct vchiq_instance *instance, int trace)
1579 {
1580 	struct vchiq_service *service;
1581 	int i;
1582 
1583 	i = 0;
1584 	rcu_read_lock();
1585 	while ((service = __next_service_by_instance(instance->state,
1586 						     instance, &i)))
1587 		service->trace = trace;
1588 	rcu_read_unlock();
1589 	instance->trace = (trace != 0);
1590 }
1591 
1592 enum vchiq_status
vchiq_use_service(unsigned int handle)1593 vchiq_use_service(unsigned int handle)
1594 {
1595 	enum vchiq_status ret = VCHIQ_ERROR;
1596 	struct vchiq_service *service = find_service_by_handle(handle);
1597 
1598 	if (service) {
1599 		ret = vchiq_use_internal(service->state, service, USE_TYPE_SERVICE);
1600 		vchiq_service_put(service);
1601 	}
1602 	return ret;
1603 }
1604 EXPORT_SYMBOL(vchiq_use_service);
1605 
1606 enum vchiq_status
vchiq_release_service(unsigned int handle)1607 vchiq_release_service(unsigned int handle)
1608 {
1609 	enum vchiq_status ret = VCHIQ_ERROR;
1610 	struct vchiq_service *service = find_service_by_handle(handle);
1611 
1612 	if (service) {
1613 		ret = vchiq_release_internal(service->state, service);
1614 		vchiq_service_put(service);
1615 	}
1616 	return ret;
1617 }
1618 EXPORT_SYMBOL(vchiq_release_service);
1619 
1620 struct service_data_struct {
1621 	int fourcc;
1622 	int clientid;
1623 	int use_count;
1624 };
1625 
1626 void
vchiq_dump_service_use_state(struct vchiq_state * state)1627 vchiq_dump_service_use_state(struct vchiq_state *state)
1628 {
1629 	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1630 	struct service_data_struct *service_data;
1631 	int i, found = 0;
1632 	/*
1633 	 * If there's more than 64 services, only dump ones with
1634 	 * non-zero counts
1635 	 */
1636 	int only_nonzero = 0;
1637 	static const char *nz = "<-- preventing suspend";
1638 
1639 	int peer_count;
1640 	int vc_use_count;
1641 	int active_services;
1642 
1643 	if (!arm_state)
1644 		return;
1645 
1646 	service_data = kmalloc_array(MAX_SERVICES, sizeof(*service_data),
1647 				     GFP_KERNEL);
1648 	if (!service_data)
1649 		return;
1650 
1651 	read_lock_bh(&arm_state->susp_res_lock);
1652 	peer_count = arm_state->peer_use_count;
1653 	vc_use_count = arm_state->videocore_use_count;
1654 	active_services = state->unused_service;
1655 	if (active_services > MAX_SERVICES)
1656 		only_nonzero = 1;
1657 
1658 	rcu_read_lock();
1659 	for (i = 0; i < active_services; i++) {
1660 		struct vchiq_service *service_ptr =
1661 			rcu_dereference(state->services[i]);
1662 
1663 		if (!service_ptr)
1664 			continue;
1665 
1666 		if (only_nonzero && !service_ptr->service_use_count)
1667 			continue;
1668 
1669 		if (service_ptr->srvstate == VCHIQ_SRVSTATE_FREE)
1670 			continue;
1671 
1672 		service_data[found].fourcc = service_ptr->base.fourcc;
1673 		service_data[found].clientid = service_ptr->client_id;
1674 		service_data[found].use_count = service_ptr->service_use_count;
1675 		found++;
1676 		if (found >= MAX_SERVICES)
1677 			break;
1678 	}
1679 	rcu_read_unlock();
1680 
1681 	read_unlock_bh(&arm_state->susp_res_lock);
1682 
1683 	if (only_nonzero)
1684 		vchiq_log_warning(vchiq_susp_log_level, "Too many active services (%d). Only dumping up to first %d services with non-zero use-count",
1685 				  active_services, found);
1686 
1687 	for (i = 0; i < found; i++) {
1688 		vchiq_log_warning(vchiq_susp_log_level, "----- %c%c%c%c:%d service count %d %s",
1689 				  VCHIQ_FOURCC_AS_4CHARS(service_data[i].fourcc),
1690 				  service_data[i].clientid, service_data[i].use_count,
1691 				  service_data[i].use_count ? nz : "");
1692 	}
1693 	vchiq_log_warning(vchiq_susp_log_level, "----- VCHIQ use count %d", peer_count);
1694 	vchiq_log_warning(vchiq_susp_log_level, "--- Overall vchiq instance use count %d",
1695 			  vc_use_count);
1696 
1697 	kfree(service_data);
1698 }
1699 
1700 enum vchiq_status
vchiq_check_service(struct vchiq_service * service)1701 vchiq_check_service(struct vchiq_service *service)
1702 {
1703 	struct vchiq_arm_state *arm_state;
1704 	enum vchiq_status ret = VCHIQ_ERROR;
1705 
1706 	if (!service || !service->state)
1707 		goto out;
1708 
1709 	arm_state = vchiq_platform_get_arm_state(service->state);
1710 
1711 	read_lock_bh(&arm_state->susp_res_lock);
1712 	if (service->service_use_count)
1713 		ret = VCHIQ_SUCCESS;
1714 	read_unlock_bh(&arm_state->susp_res_lock);
1715 
1716 	if (ret == VCHIQ_ERROR) {
1717 		vchiq_log_error(vchiq_susp_log_level,
1718 				"%s ERROR - %c%c%c%c:%d service count %d, state count %d", __func__,
1719 				VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), service->client_id,
1720 				service->service_use_count, arm_state->videocore_use_count);
1721 		vchiq_dump_service_use_state(service->state);
1722 	}
1723 out:
1724 	return ret;
1725 }
1726 
vchiq_platform_conn_state_changed(struct vchiq_state * state,enum vchiq_connstate oldstate,enum vchiq_connstate newstate)1727 void vchiq_platform_conn_state_changed(struct vchiq_state *state,
1728 				       enum vchiq_connstate oldstate,
1729 				       enum vchiq_connstate newstate)
1730 {
1731 	struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state);
1732 	char threadname[16];
1733 
1734 	vchiq_log_info(vchiq_susp_log_level, "%d: %s->%s", state->id,
1735 		       get_conn_state_name(oldstate), get_conn_state_name(newstate));
1736 	if (state->conn_state != VCHIQ_CONNSTATE_CONNECTED)
1737 		return;
1738 
1739 	write_lock_bh(&arm_state->susp_res_lock);
1740 	if (arm_state->first_connect) {
1741 		write_unlock_bh(&arm_state->susp_res_lock);
1742 		return;
1743 	}
1744 
1745 	arm_state->first_connect = 1;
1746 	write_unlock_bh(&arm_state->susp_res_lock);
1747 	snprintf(threadname, sizeof(threadname), "vchiq-keep/%d",
1748 		 state->id);
1749 	arm_state->ka_thread = kthread_create(&vchiq_keepalive_thread_func,
1750 					      (void *)state,
1751 					      threadname);
1752 	if (IS_ERR(arm_state->ka_thread)) {
1753 		vchiq_log_error(vchiq_susp_log_level,
1754 				"vchiq: FATAL: couldn't create thread %s",
1755 				threadname);
1756 	} else {
1757 		wake_up_process(arm_state->ka_thread);
1758 	}
1759 }
1760 
1761 static const struct of_device_id vchiq_of_match[] = {
1762 	{ .compatible = "brcm,bcm2835-vchiq", .data = &bcm2835_drvdata },
1763 	{ .compatible = "brcm,bcm2836-vchiq", .data = &bcm2836_drvdata },
1764 	{},
1765 };
1766 MODULE_DEVICE_TABLE(of, vchiq_of_match);
1767 
1768 static struct platform_device *
vchiq_register_child(struct platform_device * pdev,const char * name)1769 vchiq_register_child(struct platform_device *pdev, const char *name)
1770 {
1771 	struct platform_device_info pdevinfo;
1772 	struct platform_device *child;
1773 
1774 	memset(&pdevinfo, 0, sizeof(pdevinfo));
1775 
1776 	pdevinfo.parent = &pdev->dev;
1777 	pdevinfo.name = name;
1778 	pdevinfo.id = PLATFORM_DEVID_NONE;
1779 	pdevinfo.dma_mask = DMA_BIT_MASK(32);
1780 
1781 	child = platform_device_register_full(&pdevinfo);
1782 	if (IS_ERR(child)) {
1783 		dev_warn(&pdev->dev, "%s not registered\n", name);
1784 		child = NULL;
1785 	}
1786 
1787 	return child;
1788 }
1789 
vchiq_probe(struct platform_device * pdev)1790 static int vchiq_probe(struct platform_device *pdev)
1791 {
1792 	struct device_node *fw_node;
1793 	const struct of_device_id *of_id;
1794 	struct vchiq_drvdata *drvdata;
1795 	int err;
1796 
1797 	of_id = of_match_node(vchiq_of_match, pdev->dev.of_node);
1798 	drvdata = (struct vchiq_drvdata *)of_id->data;
1799 	if (!drvdata)
1800 		return -EINVAL;
1801 
1802 	fw_node = of_find_compatible_node(NULL, NULL,
1803 					  "raspberrypi,bcm2835-firmware");
1804 	if (!fw_node) {
1805 		dev_err(&pdev->dev, "Missing firmware node\n");
1806 		return -ENOENT;
1807 	}
1808 
1809 	drvdata->fw = devm_rpi_firmware_get(&pdev->dev, fw_node);
1810 	of_node_put(fw_node);
1811 	if (!drvdata->fw)
1812 		return -EPROBE_DEFER;
1813 
1814 	platform_set_drvdata(pdev, drvdata);
1815 
1816 	err = vchiq_platform_init(pdev, &g_state);
1817 	if (err)
1818 		goto failed_platform_init;
1819 
1820 	vchiq_debugfs_init();
1821 
1822 	vchiq_log_info(vchiq_arm_log_level,
1823 		       "vchiq: platform initialised - version %d (min %d)",
1824 		       VCHIQ_VERSION, VCHIQ_VERSION_MIN);
1825 
1826 	/*
1827 	 * Simply exit on error since the function handles cleanup in
1828 	 * cases of failure.
1829 	 */
1830 	err = vchiq_register_chrdev(&pdev->dev);
1831 	if (err) {
1832 		vchiq_log_warning(vchiq_arm_log_level,
1833 				  "Failed to initialize vchiq cdev");
1834 		goto error_exit;
1835 	}
1836 
1837 	bcm2835_camera = vchiq_register_child(pdev, "bcm2835-camera");
1838 	bcm2835_audio = vchiq_register_child(pdev, "bcm2835_audio");
1839 
1840 	return 0;
1841 
1842 failed_platform_init:
1843 	vchiq_log_warning(vchiq_arm_log_level, "could not initialize vchiq platform");
1844 error_exit:
1845 	return err;
1846 }
1847 
vchiq_remove(struct platform_device * pdev)1848 static int vchiq_remove(struct platform_device *pdev)
1849 {
1850 	platform_device_unregister(bcm2835_audio);
1851 	platform_device_unregister(bcm2835_camera);
1852 	vchiq_debugfs_deinit();
1853 	vchiq_deregister_chrdev();
1854 
1855 	return 0;
1856 }
1857 
1858 static struct platform_driver vchiq_driver = {
1859 	.driver = {
1860 		.name = "bcm2835_vchiq",
1861 		.of_match_table = vchiq_of_match,
1862 	},
1863 	.probe = vchiq_probe,
1864 	.remove = vchiq_remove,
1865 };
1866 
vchiq_driver_init(void)1867 static int __init vchiq_driver_init(void)
1868 {
1869 	int ret;
1870 
1871 	ret = platform_driver_register(&vchiq_driver);
1872 	if (ret)
1873 		pr_err("Failed to register vchiq driver\n");
1874 
1875 	return ret;
1876 }
1877 module_init(vchiq_driver_init);
1878 
vchiq_driver_exit(void)1879 static void __exit vchiq_driver_exit(void)
1880 {
1881 	platform_driver_unregister(&vchiq_driver);
1882 }
1883 module_exit(vchiq_driver_exit);
1884 
1885 MODULE_LICENSE("Dual BSD/GPL");
1886 MODULE_DESCRIPTION("Videocore VCHIQ driver");
1887 MODULE_AUTHOR("Broadcom Corporation");
1888