1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
4  */
5 
6 #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
7 
8 #include <linux/atomic.h>
9 #include <linux/cpu_pm.h>
10 #include <linux/delay.h>
11 #include <linux/interrupt.h>
12 #include <linux/io.h>
13 #include <linux/iopoll.h>
14 #include <linux/kernel.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/of.h>
18 #include <linux/of_irq.h>
19 #include <linux/of_platform.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23 #include <linux/wait.h>
24 
25 #include <soc/qcom/cmd-db.h>
26 #include <soc/qcom/tcs.h>
27 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
28 
29 #include "rpmh-internal.h"
30 
31 #define CREATE_TRACE_POINTS
32 #include "trace-rpmh.h"
33 
34 #define RSC_DRV_TCS_OFFSET		672
35 #define RSC_DRV_CMD_OFFSET		20
36 
37 /* DRV HW Solver Configuration Information Register */
38 #define DRV_SOLVER_CONFIG		0x04
39 #define DRV_HW_SOLVER_MASK		1
40 #define DRV_HW_SOLVER_SHIFT		24
41 
42 /* DRV TCS Configuration Information Register */
43 #define DRV_PRNT_CHLD_CONFIG		0x0C
44 #define DRV_NUM_TCS_MASK		0x3F
45 #define DRV_NUM_TCS_SHIFT		6
46 #define DRV_NCPT_MASK			0x1F
47 #define DRV_NCPT_SHIFT			27
48 
49 /* Offsets for common TCS Registers, one bit per TCS */
50 #define RSC_DRV_IRQ_ENABLE		0x00
51 #define RSC_DRV_IRQ_STATUS		0x04
52 #define RSC_DRV_IRQ_CLEAR		0x08	/* w/o; write 1 to clear */
53 
54 /*
55  * Offsets for per TCS Registers.
56  *
57  * TCSes start at 0x10 from tcs_base and are stored one after another.
58  * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one
59  * of the below to find a register.
60  */
61 #define RSC_DRV_CMD_WAIT_FOR_CMPL	0x10	/* 1 bit per command */
62 #define RSC_DRV_CONTROL			0x14
63 #define RSC_DRV_STATUS			0x18	/* zero if tcs is busy */
64 #define RSC_DRV_CMD_ENABLE		0x1C	/* 1 bit per command */
65 
66 /*
67  * Offsets for per command in a TCS.
68  *
69  * Commands (up to 16) start at 0x30 in a TCS; multiply command index
70  * by RSC_DRV_CMD_OFFSET and add one of the below to find a register.
71  */
72 #define RSC_DRV_CMD_MSGID		0x30
73 #define RSC_DRV_CMD_ADDR		0x34
74 #define RSC_DRV_CMD_DATA		0x38
75 #define RSC_DRV_CMD_STATUS		0x3C
76 #define RSC_DRV_CMD_RESP_DATA		0x40
77 
78 #define TCS_AMC_MODE_ENABLE		BIT(16)
79 #define TCS_AMC_MODE_TRIGGER		BIT(24)
80 
81 /* TCS CMD register bit mask */
82 #define CMD_MSGID_LEN			8
83 #define CMD_MSGID_RESP_REQ		BIT(8)
84 #define CMD_MSGID_WRITE			BIT(16)
85 #define CMD_STATUS_ISSUED		BIT(8)
86 #define CMD_STATUS_COMPL		BIT(16)
87 
88 /*
89  * Here's a high level overview of how all the registers in RPMH work
90  * together:
91  *
92  * - The main rpmh-rsc address is the base of a register space that can
93  *   be used to find overall configuration of the hardware
94  *   (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
95  *   space are all the TCS blocks. The offset of the TCS blocks is
96  *   specified in the device tree by "qcom,tcs-offset" and used to
97  *   compute tcs_base.
98  * - TCS blocks come one after another. Type, count, and order are
99  *   specified by the device tree as "qcom,tcs-config".
100  * - Each TCS block has some registers, then space for up to 16 commands.
101  *   Note that though address space is reserved for 16 commands, fewer
102  *   might be present. See ncpt (num cmds per TCS).
103  *
104  * Here's a picture:
105  *
106  *  +---------------------------------------------------+
107  *  |RSC                                                |
108  *  | ctrl                                              |
109  *  |                                                   |
110  *  | Drvs:                                             |
111  *  | +-----------------------------------------------+ |
112  *  | |DRV0                                           | |
113  *  | | ctrl/config                                   | |
114  *  | | IRQ                                           | |
115  *  | |                                               | |
116  *  | | TCSes:                                        | |
117  *  | | +------------------------------------------+  | |
118  *  | | |TCS0  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
119  *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
120  *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
121  *  | | +------------------------------------------+  | |
122  *  | | +------------------------------------------+  | |
123  *  | | |TCS1  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
124  *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
125  *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
126  *  | | +------------------------------------------+  | |
127  *  | | +------------------------------------------+  | |
128  *  | | |TCS2  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
129  *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
130  *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
131  *  | | +------------------------------------------+  | |
132  *  | |                    ......                     | |
133  *  | +-----------------------------------------------+ |
134  *  | +-----------------------------------------------+ |
135  *  | |DRV1                                           | |
136  *  | | (same as DRV0)                                | |
137  *  | +-----------------------------------------------+ |
138  *  |                      ......                       |
139  *  +---------------------------------------------------+
140  */
141 
142 static inline void __iomem *
tcs_reg_addr(const struct rsc_drv * drv,int reg,int tcs_id)143 tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
144 {
145 	return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg;
146 }
147 
148 static inline void __iomem *
tcs_cmd_addr(const struct rsc_drv * drv,int reg,int tcs_id,int cmd_id)149 tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
150 {
151 	return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id;
152 }
153 
read_tcs_cmd(const struct rsc_drv * drv,int reg,int tcs_id,int cmd_id)154 static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
155 			int cmd_id)
156 {
157 	return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
158 }
159 
read_tcs_reg(const struct rsc_drv * drv,int reg,int tcs_id)160 static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
161 {
162 	return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
163 }
164 
write_tcs_cmd(const struct rsc_drv * drv,int reg,int tcs_id,int cmd_id,u32 data)165 static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
166 			  int cmd_id, u32 data)
167 {
168 	writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
169 }
170 
write_tcs_reg(const struct rsc_drv * drv,int reg,int tcs_id,u32 data)171 static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
172 			  u32 data)
173 {
174 	writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
175 }
176 
write_tcs_reg_sync(const struct rsc_drv * drv,int reg,int tcs_id,u32 data)177 static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
178 			       u32 data)
179 {
180 	int i;
181 
182 	writel(data, tcs_reg_addr(drv, reg, tcs_id));
183 
184 	/*
185 	 * Wait until we read back the same value.  Use a counter rather than
186 	 * ktime for timeout since this may be called after timekeeping stops.
187 	 */
188 	for (i = 0; i < USEC_PER_SEC; i++) {
189 		if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)
190 			return;
191 		udelay(1);
192 	}
193 	pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
194 	       data, tcs_id, reg);
195 }
196 
197 /**
198  * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
199  * @drv:  The RSC controller.
200  * @type: SLEEP_TCS or WAKE_TCS
201  *
202  * This will clear the "slots" variable of the given tcs_group and also
203  * tell the hardware to forget about all entries.
204  *
205  * The caller must ensure that no other RPMH actions are happening when this
206  * function is called, since otherwise the device may immediately become
207  * used again even before this function exits.
208  */
tcs_invalidate(struct rsc_drv * drv,int type)209 static void tcs_invalidate(struct rsc_drv *drv, int type)
210 {
211 	int m;
212 	struct tcs_group *tcs = &drv->tcs[type];
213 
214 	/* Caller ensures nobody else is running so no lock */
215 	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))
216 		return;
217 
218 	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++)
219 		write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0);
220 
221 	bitmap_zero(tcs->slots, MAX_TCS_SLOTS);
222 }
223 
224 /**
225  * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
226  * @drv: The RSC controller.
227  *
228  * The caller must ensure that no other RPMH actions are happening when this
229  * function is called, since otherwise the device may immediately become
230  * used again even before this function exits.
231  */
rpmh_rsc_invalidate(struct rsc_drv * drv)232 void rpmh_rsc_invalidate(struct rsc_drv *drv)
233 {
234 	tcs_invalidate(drv, SLEEP_TCS);
235 	tcs_invalidate(drv, WAKE_TCS);
236 }
237 
238 /**
239  * get_tcs_for_msg() - Get the tcs_group used to send the given message.
240  * @drv: The RSC controller.
241  * @msg: The message we want to send.
242  *
243  * This is normally pretty straightforward except if we are trying to send
244  * an ACTIVE_ONLY message but don't have any active_only TCSes.
245  *
246  * Return: A pointer to a tcs_group or an ERR_PTR.
247  */
get_tcs_for_msg(struct rsc_drv * drv,const struct tcs_request * msg)248 static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
249 					 const struct tcs_request *msg)
250 {
251 	int type;
252 	struct tcs_group *tcs;
253 
254 	switch (msg->state) {
255 	case RPMH_ACTIVE_ONLY_STATE:
256 		type = ACTIVE_TCS;
257 		break;
258 	case RPMH_WAKE_ONLY_STATE:
259 		type = WAKE_TCS;
260 		break;
261 	case RPMH_SLEEP_STATE:
262 		type = SLEEP_TCS;
263 		break;
264 	default:
265 		return ERR_PTR(-EINVAL);
266 	}
267 
268 	/*
269 	 * If we are making an active request on a RSC that does not have a
270 	 * dedicated TCS for active state use, then re-purpose a wake TCS to
271 	 * send active votes. This is safe because we ensure any active-only
272 	 * transfers have finished before we use it (maybe by running from
273 	 * the last CPU in PM code).
274 	 */
275 	tcs = &drv->tcs[type];
276 	if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
277 		tcs = &drv->tcs[WAKE_TCS];
278 
279 	return tcs;
280 }
281 
282 /**
283  * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
284  * @drv:    The RSC controller.
285  * @tcs_id: The global ID of this TCS.
286  *
287  * For ACTIVE_ONLY transfers we want to call back into the client when the
288  * transfer finishes. To do this we need the "request" that the client
289  * originally provided us. This function grabs the request that we stashed
290  * when we started the transfer.
291  *
292  * This only makes sense for ACTIVE_ONLY transfers since those are the only
293  * ones we track sending (the only ones we enable interrupts for and the only
294  * ones we call back to the client for).
295  *
296  * Return: The stashed request.
297  */
get_req_from_tcs(struct rsc_drv * drv,int tcs_id)298 static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
299 						  int tcs_id)
300 {
301 	struct tcs_group *tcs;
302 	int i;
303 
304 	for (i = 0; i < TCS_TYPE_NR; i++) {
305 		tcs = &drv->tcs[i];
306 		if (tcs->mask & BIT(tcs_id))
307 			return tcs->req[tcs_id - tcs->offset];
308 	}
309 
310 	return NULL;
311 }
312 
313 /**
314  * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
315  * @drv:     The controller.
316  * @tcs_id:  The global ID of this TCS.
317  * @trigger: If true then untrigger/retrigger. If false then just untrigger.
318  *
319  * In the normal case we only ever call with "trigger=true" to start a
320  * transfer. That will un-trigger/disable the TCS from the last transfer
321  * then trigger/enable for this transfer.
322  *
323  * If we borrowed a wake TCS for an active-only transfer we'll also call
324  * this function with "trigger=false" to just do the un-trigger/disable
325  * before using the TCS for wake purposes again.
326  *
327  * Note that the AP is only in charge of triggering active-only transfers.
328  * The AP never triggers sleep/wake values using this function.
329  */
__tcs_set_trigger(struct rsc_drv * drv,int tcs_id,bool trigger)330 static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
331 {
332 	u32 enable;
333 
334 	/*
335 	 * HW req: Clear the DRV_CONTROL and enable TCS again
336 	 * While clearing ensure that the AMC mode trigger is cleared
337 	 * and then the mode enable is cleared.
338 	 */
339 	enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id);
340 	enable &= ~TCS_AMC_MODE_TRIGGER;
341 	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
342 	enable &= ~TCS_AMC_MODE_ENABLE;
343 	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
344 
345 	if (trigger) {
346 		/* Enable the AMC mode on the TCS and then trigger the TCS */
347 		enable = TCS_AMC_MODE_ENABLE;
348 		write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
349 		enable |= TCS_AMC_MODE_TRIGGER;
350 		write_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, enable);
351 	}
352 }
353 
354 /**
355  * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
356  * @drv:     The controller.
357  * @tcs_id:  The global ID of this TCS.
358  * @enable:  If true then enable; if false then disable
359  *
360  * We only ever call this when we borrow a wake TCS for an active-only
361  * transfer. For active-only TCSes interrupts are always left enabled.
362  */
enable_tcs_irq(struct rsc_drv * drv,int tcs_id,bool enable)363 static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
364 {
365 	u32 data;
366 
367 	data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE);
368 	if (enable)
369 		data |= BIT(tcs_id);
370 	else
371 		data &= ~BIT(tcs_id);
372 	writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE);
373 }
374 
375 /**
376  * tcs_tx_done() - TX Done interrupt handler.
377  * @irq: The IRQ number (ignored).
378  * @p:   Pointer to "struct rsc_drv".
379  *
380  * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
381  * IRQ for) when a transfer is done.
382  *
383  * Return: IRQ_HANDLED
384  */
tcs_tx_done(int irq,void * p)385 static irqreturn_t tcs_tx_done(int irq, void *p)
386 {
387 	struct rsc_drv *drv = p;
388 	int i, j, err = 0;
389 	unsigned long irq_status;
390 	const struct tcs_request *req;
391 	struct tcs_cmd *cmd;
392 
393 	irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS);
394 
395 	for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) {
396 		req = get_req_from_tcs(drv, i);
397 		if (WARN_ON(!req))
398 			goto skip;
399 
400 		err = 0;
401 		for (j = 0; j < req->num_cmds; j++) {
402 			u32 sts;
403 
404 			cmd = &req->cmds[j];
405 			sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j);
406 			if (!(sts & CMD_STATUS_ISSUED) ||
407 			   ((req->wait_for_compl || cmd->wait) &&
408 			   !(sts & CMD_STATUS_COMPL))) {
409 				pr_err("Incomplete request: %s: addr=%#x data=%#x",
410 				       drv->name, cmd->addr, cmd->data);
411 				err = -EIO;
412 			}
413 		}
414 
415 		trace_rpmh_tx_done(drv, i, req, err);
416 
417 		/*
418 		 * If wake tcs was re-purposed for sending active
419 		 * votes, clear AMC trigger & enable modes and
420 		 * disable interrupt for this TCS
421 		 */
422 		if (!drv->tcs[ACTIVE_TCS].num_tcs)
423 			__tcs_set_trigger(drv, i, false);
424 skip:
425 		/* Reclaim the TCS */
426 		write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0);
427 		writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR);
428 		spin_lock(&drv->lock);
429 		clear_bit(i, drv->tcs_in_use);
430 		/*
431 		 * Disable interrupt for WAKE TCS to avoid being
432 		 * spammed with interrupts coming when the solver
433 		 * sends its wake votes.
434 		 */
435 		if (!drv->tcs[ACTIVE_TCS].num_tcs)
436 			enable_tcs_irq(drv, i, false);
437 		spin_unlock(&drv->lock);
438 		wake_up(&drv->tcs_wait);
439 		if (req)
440 			rpmh_tx_done(req, err);
441 	}
442 
443 	return IRQ_HANDLED;
444 }
445 
446 /**
447  * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
448  * @drv:    The controller.
449  * @tcs_id: The global ID of this TCS.
450  * @cmd_id: The index within the TCS to start writing.
451  * @msg:    The message we want to send, which will contain several addr/data
452  *          pairs to program (but few enough that they all fit in one TCS).
453  *
454  * This is used for all types of transfers (active, sleep, and wake).
455  */
__tcs_buffer_write(struct rsc_drv * drv,int tcs_id,int cmd_id,const struct tcs_request * msg)456 static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
457 			       const struct tcs_request *msg)
458 {
459 	u32 msgid;
460 	u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE;
461 	u32 cmd_enable = 0;
462 	struct tcs_cmd *cmd;
463 	int i, j;
464 
465 	/* Convert all commands to RR when the request has wait_for_compl set */
466 	cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
467 
468 	for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
469 		cmd = &msg->cmds[i];
470 		cmd_enable |= BIT(j);
471 		msgid = cmd_msgid;
472 		/*
473 		 * Additionally, if the cmd->wait is set, make the command
474 		 * response reqd even if the overall request was fire-n-forget.
475 		 */
476 		msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;
477 
478 		write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid);
479 		write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr);
480 		write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data);
481 		trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd);
482 	}
483 
484 	cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);
485 	write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable);
486 }
487 
488 /**
489  * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
490  * @drv: The controller.
491  * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
492  * @msg: The message we want to send, which will contain several addr/data
493  *       pairs to program (but few enough that they all fit in one TCS).
494  *
495  * This will walk through the TCSes in the group and check if any of them
496  * appear to be sending to addresses referenced in the message. If it finds
497  * one it'll return -EBUSY.
498  *
499  * Only for use for active-only transfers.
500  *
501  * Must be called with the drv->lock held since that protects tcs_in_use.
502  *
503  * Return: 0 if nothing in flight or -EBUSY if we should try again later.
504  *         The caller must re-enable interrupts between tries since that's
505  *         the only way tcs_in_use will ever be updated and the only way
506  *         RSC_DRV_CMD_ENABLE will ever be cleared.
507  */
check_for_req_inflight(struct rsc_drv * drv,struct tcs_group * tcs,const struct tcs_request * msg)508 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
509 				  const struct tcs_request *msg)
510 {
511 	unsigned long curr_enabled;
512 	u32 addr;
513 	int j, k;
514 	int i = tcs->offset;
515 
516 	for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {
517 		curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i);
518 
519 		for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) {
520 			addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, i, j);
521 			for (k = 0; k < msg->num_cmds; k++) {
522 				if (addr == msg->cmds[k].addr)
523 					return -EBUSY;
524 			}
525 		}
526 	}
527 
528 	return 0;
529 }
530 
531 /**
532  * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
533  * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
534  *       we borrowed it because there are zero active-only ones).
535  *
536  * Must be called with the drv->lock held since that protects tcs_in_use.
537  *
538  * Return: The first tcs that's free or -EBUSY if all in use.
539  */
find_free_tcs(struct tcs_group * tcs)540 static int find_free_tcs(struct tcs_group *tcs)
541 {
542 	const struct rsc_drv *drv = tcs->drv;
543 	unsigned long i;
544 	unsigned long max = tcs->offset + tcs->num_tcs;
545 
546 	i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);
547 	if (i >= max)
548 		return -EBUSY;
549 
550 	return i;
551 }
552 
553 /**
554  * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
555  * @drv: The controller.
556  * @tcs: The tcs_group used for ACTIVE_ONLY transfers.
557  * @msg: The data to be sent.
558  *
559  * Claims a tcs in the given tcs_group while making sure that no existing cmd
560  * is in flight that would conflict with the one in @msg.
561  *
562  * Context: Must be called with the drv->lock held since that protects
563  * tcs_in_use.
564  *
565  * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
566  * or the tcs_group is full.
567  */
claim_tcs_for_req(struct rsc_drv * drv,struct tcs_group * tcs,const struct tcs_request * msg)568 static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,
569 			     const struct tcs_request *msg)
570 {
571 	int ret;
572 
573 	/*
574 	 * The h/w does not like if we send a request to the same address,
575 	 * when one is already in-flight or being processed.
576 	 */
577 	ret = check_for_req_inflight(drv, tcs, msg);
578 	if (ret)
579 		return ret;
580 
581 	return find_free_tcs(tcs);
582 }
583 
584 /**
585  * rpmh_rsc_send_data() - Write / trigger active-only message.
586  * @drv: The controller.
587  * @msg: The data to be sent.
588  *
589  * NOTES:
590  * - This is only used for "ACTIVE_ONLY" since the limitations of this
591  *   function don't make sense for sleep/wake cases.
592  * - To do the transfer, we will grab a whole TCS for ourselves--we don't
593  *   try to share. If there are none available we'll wait indefinitely
594  *   for a free one.
595  * - This function will not wait for the commands to be finished, only for
596  *   data to be programmed into the RPMh. See rpmh_tx_done() which will
597  *   be called when the transfer is fully complete.
598  * - This function must be called with interrupts enabled. If the hardware
599  *   is busy doing someone else's transfer we need that transfer to fully
600  *   finish so that we can have the hardware, and to fully finish it needs
601  *   the interrupt handler to run. If the interrupts is set to run on the
602  *   active CPU this can never happen if interrupts are disabled.
603  *
604  * Return: 0 on success, -EINVAL on error.
605  */
rpmh_rsc_send_data(struct rsc_drv * drv,const struct tcs_request * msg)606 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
607 {
608 	struct tcs_group *tcs;
609 	int tcs_id;
610 	unsigned long flags;
611 
612 	tcs = get_tcs_for_msg(drv, msg);
613 	if (IS_ERR(tcs))
614 		return PTR_ERR(tcs);
615 
616 	spin_lock_irqsave(&drv->lock, flags);
617 
618 	/* Wait forever for a free tcs. It better be there eventually! */
619 	wait_event_lock_irq(drv->tcs_wait,
620 			    (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,
621 			    drv->lock);
622 
623 	tcs->req[tcs_id - tcs->offset] = msg;
624 	set_bit(tcs_id, drv->tcs_in_use);
625 	if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {
626 		/*
627 		 * Clear previously programmed WAKE commands in selected
628 		 * repurposed TCS to avoid triggering them. tcs->slots will be
629 		 * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate()
630 		 */
631 		write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);
632 		enable_tcs_irq(drv, tcs_id, true);
633 	}
634 	spin_unlock_irqrestore(&drv->lock, flags);
635 
636 	/*
637 	 * These two can be done after the lock is released because:
638 	 * - We marked "tcs_in_use" under lock.
639 	 * - Once "tcs_in_use" has been marked nobody else could be writing
640 	 *   to these registers until the interrupt goes off.
641 	 * - The interrupt can't go off until we trigger w/ the last line
642 	 *   of __tcs_set_trigger() below.
643 	 */
644 	__tcs_buffer_write(drv, tcs_id, 0, msg);
645 	__tcs_set_trigger(drv, tcs_id, true);
646 
647 	return 0;
648 }
649 
650 /**
651  * find_slots() - Find a place to write the given message.
652  * @tcs:    The tcs group to search.
653  * @msg:    The message we want to find room for.
654  * @tcs_id: If we return 0 from the function, we return the global ID of the
655  *          TCS to write to here.
656  * @cmd_id: If we return 0 from the function, we return the index of
657  *          the command array of the returned TCS where the client should
658  *          start writing the message.
659  *
660  * Only for use on sleep/wake TCSes since those are the only ones we maintain
661  * tcs->slots for.
662  *
663  * Return: -ENOMEM if there was no room, else 0.
664  */
find_slots(struct tcs_group * tcs,const struct tcs_request * msg,int * tcs_id,int * cmd_id)665 static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
666 		      int *tcs_id, int *cmd_id)
667 {
668 	int slot, offset;
669 	int i = 0;
670 
671 	/* Do over, until we can fit the full payload in a single TCS */
672 	do {
673 		slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,
674 						  i, msg->num_cmds, 0);
675 		if (slot >= tcs->num_tcs * tcs->ncpt)
676 			return -ENOMEM;
677 		i += tcs->ncpt;
678 	} while (slot + msg->num_cmds - 1 >= i);
679 
680 	bitmap_set(tcs->slots, slot, msg->num_cmds);
681 
682 	offset = slot / tcs->ncpt;
683 	*tcs_id = offset + tcs->offset;
684 	*cmd_id = slot % tcs->ncpt;
685 
686 	return 0;
687 }
688 
689 /**
690  * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
691  * @drv: The controller.
692  * @msg: The data to be written to the controller.
693  *
694  * This should only be called for sleep/wake state, never active-only
695  * state.
696  *
697  * The caller must ensure that no other RPMH actions are happening and the
698  * controller is idle when this function is called since it runs lockless.
699  *
700  * Return: 0 if no error; else -error.
701  */
rpmh_rsc_write_ctrl_data(struct rsc_drv * drv,const struct tcs_request * msg)702 int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
703 {
704 	struct tcs_group *tcs;
705 	int tcs_id = 0, cmd_id = 0;
706 	int ret;
707 
708 	tcs = get_tcs_for_msg(drv, msg);
709 	if (IS_ERR(tcs))
710 		return PTR_ERR(tcs);
711 
712 	/* find the TCS id and the command in the TCS to write to */
713 	ret = find_slots(tcs, msg, &tcs_id, &cmd_id);
714 	if (!ret)
715 		__tcs_buffer_write(drv, tcs_id, cmd_id, msg);
716 
717 	return ret;
718 }
719 
720 /**
721  * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
722  * @drv: The controller
723  *
724  * Checks if any of the AMCs are busy in handling ACTIVE sets.
725  * This is called from the last cpu powering down before flushing
726  * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
727  * power collapse, so deny from the last cpu's pm notification.
728  *
729  * Context: Must be called with the drv->lock held.
730  *
731  * Return:
732  * * False		- AMCs are idle
733  * * True		- AMCs are busy
734  */
rpmh_rsc_ctrlr_is_busy(struct rsc_drv * drv)735 static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
736 {
737 	unsigned long set;
738 	const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];
739 	unsigned long max;
740 
741 	/*
742 	 * If we made an active request on a RSC that does not have a
743 	 * dedicated TCS for active state use, then re-purposed wake TCSes
744 	 * should be checked for not busy, because we used wake TCSes for
745 	 * active requests in this case.
746 	 */
747 	if (!tcs->num_tcs)
748 		tcs = &drv->tcs[WAKE_TCS];
749 
750 	max = tcs->offset + tcs->num_tcs;
751 	set = find_next_bit(drv->tcs_in_use, max, tcs->offset);
752 
753 	return set < max;
754 }
755 
756 /**
757  * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
758  * @nfb:    Pointer to the notifier block in struct rsc_drv.
759  * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
760  * @v:      Unused
761  *
762  * This function is given to cpu_pm_register_notifier so we can be informed
763  * about when CPUs go down. When all CPUs go down we know no more active
764  * transfers will be started so we write sleep/wake sets. This function gets
765  * called from cpuidle code paths and also at system suspend time.
766  *
767  * If its last CPU going down and AMCs are not busy then writes cached sleep
768  * and wake messages to TCSes. The firmware then takes care of triggering
769  * them when entering deepest low power modes.
770  *
771  * Return: See cpu_pm_register_notifier()
772  */
rpmh_rsc_cpu_pm_callback(struct notifier_block * nfb,unsigned long action,void * v)773 static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
774 				    unsigned long action, void *v)
775 {
776 	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
777 	int ret = NOTIFY_OK;
778 	int cpus_in_pm;
779 
780 	switch (action) {
781 	case CPU_PM_ENTER:
782 		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
783 		/*
784 		 * NOTE: comments for num_online_cpus() point out that it's
785 		 * only a snapshot so we need to be careful. It should be OK
786 		 * for us to use, though.  It's important for us not to miss
787 		 * if we're the last CPU going down so it would only be a
788 		 * problem if a CPU went offline right after we did the check
789 		 * AND that CPU was not idle AND that CPU was the last non-idle
790 		 * CPU. That can't happen. CPUs would have to come out of idle
791 		 * before the CPU could go offline.
792 		 */
793 		if (cpus_in_pm < num_online_cpus())
794 			return NOTIFY_OK;
795 		break;
796 	case CPU_PM_ENTER_FAILED:
797 	case CPU_PM_EXIT:
798 		atomic_dec(&drv->cpus_in_pm);
799 		return NOTIFY_OK;
800 	default:
801 		return NOTIFY_DONE;
802 	}
803 
804 	/*
805 	 * It's likely we're on the last CPU. Grab the drv->lock and write
806 	 * out the sleep/wake commands to RPMH hardware. Grabbing the lock
807 	 * means that if we race with another CPU coming up we are still
808 	 * guaranteed to be safe. If another CPU came up just after we checked
809 	 * and has grabbed the lock or started an active transfer then we'll
810 	 * notice we're busy and abort. If another CPU comes up after we start
811 	 * flushing it will be blocked from starting an active transfer until
812 	 * we're done flushing. If another CPU starts an active transfer after
813 	 * we release the lock we're still OK because we're no longer the last
814 	 * CPU.
815 	 */
816 	if (spin_trylock(&drv->lock)) {
817 		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))
818 			ret = NOTIFY_BAD;
819 		spin_unlock(&drv->lock);
820 	} else {
821 		/* Another CPU must be up */
822 		return NOTIFY_OK;
823 	}
824 
825 	if (ret == NOTIFY_BAD) {
826 		/* Double-check if we're here because someone else is up */
827 		if (cpus_in_pm < num_online_cpus())
828 			ret = NOTIFY_OK;
829 		else
830 			/* We won't be called w/ CPU_PM_ENTER_FAILED */
831 			atomic_dec(&drv->cpus_in_pm);
832 	}
833 
834 	return ret;
835 }
836 
rpmh_probe_tcs_config(struct platform_device * pdev,struct rsc_drv * drv,void __iomem * base)837 static int rpmh_probe_tcs_config(struct platform_device *pdev,
838 				 struct rsc_drv *drv, void __iomem *base)
839 {
840 	struct tcs_type_config {
841 		u32 type;
842 		u32 n;
843 	} tcs_cfg[TCS_TYPE_NR] = { { 0 } };
844 	struct device_node *dn = pdev->dev.of_node;
845 	u32 config, max_tcs, ncpt, offset;
846 	int i, ret, n, st = 0;
847 	struct tcs_group *tcs;
848 
849 	ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);
850 	if (ret)
851 		return ret;
852 	drv->tcs_base = base + offset;
853 
854 	config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG);
855 
856 	max_tcs = config;
857 	max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);
858 	max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);
859 
860 	ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT);
861 	ncpt = ncpt >> DRV_NCPT_SHIFT;
862 
863 	n = of_property_count_u32_elems(dn, "qcom,tcs-config");
864 	if (n != 2 * TCS_TYPE_NR)
865 		return -EINVAL;
866 
867 	for (i = 0; i < TCS_TYPE_NR; i++) {
868 		ret = of_property_read_u32_index(dn, "qcom,tcs-config",
869 						 i * 2, &tcs_cfg[i].type);
870 		if (ret)
871 			return ret;
872 		if (tcs_cfg[i].type >= TCS_TYPE_NR)
873 			return -EINVAL;
874 
875 		ret = of_property_read_u32_index(dn, "qcom,tcs-config",
876 						 i * 2 + 1, &tcs_cfg[i].n);
877 		if (ret)
878 			return ret;
879 		if (tcs_cfg[i].n > MAX_TCS_PER_TYPE)
880 			return -EINVAL;
881 	}
882 
883 	for (i = 0; i < TCS_TYPE_NR; i++) {
884 		tcs = &drv->tcs[tcs_cfg[i].type];
885 		if (tcs->drv)
886 			return -EINVAL;
887 		tcs->drv = drv;
888 		tcs->type = tcs_cfg[i].type;
889 		tcs->num_tcs = tcs_cfg[i].n;
890 		tcs->ncpt = ncpt;
891 
892 		if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
893 			continue;
894 
895 		if (st + tcs->num_tcs > max_tcs ||
896 		    st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))
897 			return -EINVAL;
898 
899 		tcs->mask = ((1 << tcs->num_tcs) - 1) << st;
900 		tcs->offset = st;
901 		st += tcs->num_tcs;
902 	}
903 
904 	drv->num_tcs = st;
905 
906 	return 0;
907 }
908 
rpmh_rsc_probe(struct platform_device * pdev)909 static int rpmh_rsc_probe(struct platform_device *pdev)
910 {
911 	struct device_node *dn = pdev->dev.of_node;
912 	struct rsc_drv *drv;
913 	char drv_id[10] = {0};
914 	int ret, irq;
915 	u32 solver_config;
916 	void __iomem *base;
917 
918 	/*
919 	 * Even though RPMh doesn't directly use cmd-db, all of its children
920 	 * do. To avoid adding this check to our children we'll do it now.
921 	 */
922 	ret = cmd_db_ready();
923 	if (ret) {
924 		if (ret != -EPROBE_DEFER)
925 			dev_err(&pdev->dev, "Command DB not available (%d)\n",
926 									ret);
927 		return ret;
928 	}
929 
930 	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);
931 	if (!drv)
932 		return -ENOMEM;
933 
934 	ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);
935 	if (ret)
936 		return ret;
937 
938 	drv->name = of_get_property(dn, "label", NULL);
939 	if (!drv->name)
940 		drv->name = dev_name(&pdev->dev);
941 
942 	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
943 	base = devm_platform_ioremap_resource_byname(pdev, drv_id);
944 	if (IS_ERR(base))
945 		return PTR_ERR(base);
946 
947 	ret = rpmh_probe_tcs_config(pdev, drv, base);
948 	if (ret)
949 		return ret;
950 
951 	spin_lock_init(&drv->lock);
952 	init_waitqueue_head(&drv->tcs_wait);
953 	bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);
954 
955 	irq = platform_get_irq(pdev, drv->id);
956 	if (irq < 0)
957 		return irq;
958 
959 	ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done,
960 			       IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND,
961 			       drv->name, drv);
962 	if (ret)
963 		return ret;
964 
965 	/*
966 	 * CPU PM notification are not required for controllers that support
967 	 * 'HW solver' mode where they can be in autonomous mode executing low
968 	 * power mode to power down.
969 	 */
970 	solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG);
971 	solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
972 	solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
973 	if (!solver_config) {
974 		drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
975 		cpu_pm_register_notifier(&drv->rsc_pm);
976 	}
977 
978 	/* Enable the active TCS to send requests immediately */
979 	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,
980 		       drv->tcs_base + RSC_DRV_IRQ_ENABLE);
981 
982 	spin_lock_init(&drv->client.cache_lock);
983 	INIT_LIST_HEAD(&drv->client.cache);
984 	INIT_LIST_HEAD(&drv->client.batch_cache);
985 
986 	dev_set_drvdata(&pdev->dev, drv);
987 
988 	return devm_of_platform_populate(&pdev->dev);
989 }
990 
991 static const struct of_device_id rpmh_drv_match[] = {
992 	{ .compatible = "qcom,rpmh-rsc", },
993 	{ }
994 };
995 MODULE_DEVICE_TABLE(of, rpmh_drv_match);
996 
997 static struct platform_driver rpmh_driver = {
998 	.probe = rpmh_rsc_probe,
999 	.driver = {
1000 		  .name = "rpmh",
1001 		  .of_match_table = rpmh_drv_match,
1002 		  .suppress_bind_attrs = true,
1003 	},
1004 };
1005 
rpmh_driver_init(void)1006 static int __init rpmh_driver_init(void)
1007 {
1008 	return platform_driver_register(&rpmh_driver);
1009 }
1010 arch_initcall(rpmh_driver_init);
1011 
1012 MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver");
1013 MODULE_LICENSE("GPL v2");
1014