1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * hcd_queue.c - DesignWare HS OTG Controller host queuing routines
4  *
5  * Copyright (C) 2004-2013 Synopsys, Inc.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions, and the following disclaimer,
12  *    without modification.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. The names of the above-listed copyright holders may not be used
17  *    to endorse or promote products derived from this software without
18  *    specific prior written permission.
19  *
20  * ALTERNATIVELY, this software may be distributed under the terms of the
21  * GNU General Public License ("GPL") as published by the Free Software
22  * Foundation; either version 2 of the License, or (at your option) any
23  * later version.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
26  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
27  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36  */
37 
38 /*
39  * This file contains the functions to manage Queue Heads and Queue
40  * Transfer Descriptors for Host mode
41  */
42 #include <linux/gcd.h>
43 #include <linux/kernel.h>
44 #include <linux/module.h>
45 #include <linux/spinlock.h>
46 #include <linux/interrupt.h>
47 #include <linux/dma-mapping.h>
48 #include <linux/io.h>
49 #include <linux/slab.h>
50 #include <linux/usb.h>
51 
52 #include <linux/usb/hcd.h>
53 #include <linux/usb/ch11.h>
54 
55 #include "core.h"
56 #include "hcd.h"
57 
58 /* Wait this long before releasing periodic reservation */
59 #define DWC2_UNRESERVE_DELAY (msecs_to_jiffies(5))
60 
61 /* If we get a NAK, wait this long before retrying */
62 #define DWC2_RETRY_WAIT_DELAY (1 * NSEC_PER_MSEC)
63 
64 /**
65  * dwc2_periodic_channel_available() - Checks that a channel is available for a
66  * periodic transfer
67  *
68  * @hsotg: The HCD state structure for the DWC OTG controller
69  *
70  * Return: 0 if successful, negative error code otherwise
71  */
dwc2_periodic_channel_available(struct dwc2_hsotg * hsotg)72 static int dwc2_periodic_channel_available(struct dwc2_hsotg *hsotg)
73 {
74 	/*
75 	 * Currently assuming that there is a dedicated host channel for
76 	 * each periodic transaction plus at least one host channel for
77 	 * non-periodic transactions
78 	 */
79 	int status;
80 	int num_channels;
81 
82 	num_channels = hsotg->params.host_channels;
83 	if ((hsotg->periodic_channels + hsotg->non_periodic_channels <
84 	     num_channels) && (hsotg->periodic_channels < num_channels - 1)) {
85 		status = 0;
86 	} else {
87 		dev_dbg(hsotg->dev,
88 			"%s: Total channels: %d, Periodic: %d, Non-periodic: %d\n",
89 			__func__, num_channels,
90 			hsotg->periodic_channels, hsotg->non_periodic_channels);
91 		status = -ENOSPC;
92 	}
93 
94 	return status;
95 }
96 
97 /**
98  * dwc2_check_periodic_bandwidth() - Checks that there is sufficient bandwidth
99  * for the specified QH in the periodic schedule
100  *
101  * @hsotg: The HCD state structure for the DWC OTG controller
102  * @qh:    QH containing periodic bandwidth required
103  *
104  * Return: 0 if successful, negative error code otherwise
105  *
106  * For simplicity, this calculation assumes that all the transfers in the
107  * periodic schedule may occur in the same (micro)frame
108  */
dwc2_check_periodic_bandwidth(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)109 static int dwc2_check_periodic_bandwidth(struct dwc2_hsotg *hsotg,
110 					 struct dwc2_qh *qh)
111 {
112 	int status;
113 	s16 max_claimed_usecs;
114 
115 	status = 0;
116 
117 	if (qh->dev_speed == USB_SPEED_HIGH || qh->do_split) {
118 		/*
119 		 * High speed mode
120 		 * Max periodic usecs is 80% x 125 usec = 100 usec
121 		 */
122 		max_claimed_usecs = 100 - qh->host_us;
123 	} else {
124 		/*
125 		 * Full speed mode
126 		 * Max periodic usecs is 90% x 1000 usec = 900 usec
127 		 */
128 		max_claimed_usecs = 900 - qh->host_us;
129 	}
130 
131 	if (hsotg->periodic_usecs > max_claimed_usecs) {
132 		dev_err(hsotg->dev,
133 			"%s: already claimed usecs %d, required usecs %d\n",
134 			__func__, hsotg->periodic_usecs, qh->host_us);
135 		status = -ENOSPC;
136 	}
137 
138 	return status;
139 }
140 
141 /**
142  * pmap_schedule() - Schedule time in a periodic bitmap (pmap).
143  *
144  * @map:             The bitmap representing the schedule; will be updated
145  *                   upon success.
146  * @bits_per_period: The schedule represents several periods.  This is how many
147  *                   bits are in each period.  It's assumed that the beginning
148  *                   of the schedule will repeat after its end.
149  * @periods_in_map:  The number of periods in the schedule.
150  * @num_bits:        The number of bits we need per period we want to reserve
151  *                   in this function call.
152  * @interval:        How often we need to be scheduled for the reservation this
153  *                   time.  1 means every period.  2 means every other period.
154  *                   ...you get the picture?
155  * @start:           The bit number to start at.  Normally 0.  Must be within
156  *                   the interval or we return failure right away.
157  * @only_one_period: Normally we'll allow picking a start anywhere within the
158  *                   first interval, since we can still make all repetition
159  *                   requirements by doing that.  However, if you pass true
160  *                   here then we'll return failure if we can't fit within
161  *                   the period that "start" is in.
162  *
163  * The idea here is that we want to schedule time for repeating events that all
164  * want the same resource.  The resource is divided into fixed-sized periods
165  * and the events want to repeat every "interval" periods.  The schedule
166  * granularity is one bit.
167  *
168  * To keep things "simple", we'll represent our schedule with a bitmap that
169  * contains a fixed number of periods.  This gets rid of a lot of complexity
170  * but does mean that we need to handle things specially (and non-ideally) if
171  * the number of the periods in the schedule doesn't match well with the
172  * intervals that we're trying to schedule.
173  *
174  * Here's an explanation of the scheme we'll implement, assuming 8 periods.
175  * - If interval is 1, we need to take up space in each of the 8
176  *   periods we're scheduling.  Easy.
177  * - If interval is 2, we need to take up space in half of the
178  *   periods.  Again, easy.
179  * - If interval is 3, we actually need to fall back to interval 1.
180  *   Why?  Because we might need time in any period.  AKA for the
181  *   first 8 periods, we'll be in slot 0, 3, 6.  Then we'll be
182  *   in slot 1, 4, 7.  Then we'll be in 2, 5.  Then we'll be back to
183  *   0, 3, and 6.  Since we could be in any frame we need to reserve
184  *   for all of them.  Sucks, but that's what you gotta do.  Note that
185  *   if we were instead scheduling 8 * 3 = 24 we'd do much better, but
186  *   then we need more memory and time to do scheduling.
187  * - If interval is 4, easy.
188  * - If interval is 5, we again need interval 1.  The schedule will be
189  *   0, 5, 2, 7, 4, 1, 6, 3, 0
190  * - If interval is 6, we need interval 2.  0, 6, 4, 2.
191  * - If interval is 7, we need interval 1.
192  * - If interval is 8, we need interval 8.
193  *
194  * If you do the math, you'll see that we need to pretend that interval is
195  * equal to the greatest_common_divisor(interval, periods_in_map).
196  *
197  * Note that at the moment this function tends to front-pack the schedule.
198  * In some cases that's really non-ideal (it's hard to schedule things that
199  * need to repeat every period).  In other cases it's perfect (you can easily
200  * schedule bigger, less often repeating things).
201  *
202  * Here's the algorithm in action (8 periods, 5 bits per period):
203  *  |**   |     |**   |     |**   |     |**   |     |   OK 2 bits, intv 2 at 0
204  *  |*****|  ***|*****|  ***|*****|  ***|*****|  ***|   OK 3 bits, intv 3 at 2
205  *  |*****|* ***|*****|  ***|*****|* ***|*****|  ***|   OK 1 bits, intv 4 at 5
206  *  |**   |*    |**   |     |**   |*    |**   |     | Remv 3 bits, intv 3 at 2
207  *  |***  |*    |***  |     |***  |*    |***  |     |   OK 1 bits, intv 6 at 2
208  *  |**** |*  * |**** |   * |**** |*  * |**** |   * |   OK 1 bits, intv 1 at 3
209  *  |**** |**** |**** | *** |**** |**** |**** | *** |   OK 2 bits, intv 2 at 6
210  *  |*****|*****|*****| ****|*****|*****|*****| ****|   OK 1 bits, intv 1 at 4
211  *  |*****|*****|*****| ****|*****|*****|*****| ****| FAIL 1 bits, intv 1
212  *  |  ***|*****|  ***| ****|  ***|*****|  ***| ****| Remv 2 bits, intv 2 at 0
213  *  |  ***| ****|  ***| ****|  ***| ****|  ***| ****| Remv 1 bits, intv 4 at 5
214  *  |   **| ****|   **| ****|   **| ****|   **| ****| Remv 1 bits, intv 6 at 2
215  *  |    *| ** *|    *| ** *|    *| ** *|    *| ** *| Remv 1 bits, intv 1 at 3
216  *  |    *|    *|    *|    *|    *|    *|    *|    *| Remv 2 bits, intv 2 at 6
217  *  |     |     |     |     |     |     |     |     | Remv 1 bits, intv 1 at 4
218  *  |**   |     |**   |     |**   |     |**   |     |   OK 2 bits, intv 2 at 0
219  *  |***  |     |**   |     |***  |     |**   |     |   OK 1 bits, intv 4 at 2
220  *  |*****|     |** **|     |*****|     |** **|     |   OK 2 bits, intv 2 at 3
221  *  |*****|*    |** **|     |*****|*    |** **|     |   OK 1 bits, intv 4 at 5
222  *  |*****|***  |** **| **  |*****|***  |** **| **  |   OK 2 bits, intv 2 at 6
223  *  |*****|*****|** **| ****|*****|*****|** **| ****|   OK 2 bits, intv 2 at 8
224  *  |*****|*****|*****| ****|*****|*****|*****| ****|   OK 1 bits, intv 4 at 12
225  *
226  * This function is pretty generic and could be easily abstracted if anything
227  * needed similar scheduling.
228  *
229  * Returns either -ENOSPC or a >= 0 start bit which should be passed to the
230  * unschedule routine.  The map bitmap will be updated on a non-error result.
231  */
pmap_schedule(unsigned long * map,int bits_per_period,int periods_in_map,int num_bits,int interval,int start,bool only_one_period)232 static int pmap_schedule(unsigned long *map, int bits_per_period,
233 			 int periods_in_map, int num_bits,
234 			 int interval, int start, bool only_one_period)
235 {
236 	int interval_bits;
237 	int to_reserve;
238 	int first_end;
239 	int i;
240 
241 	if (num_bits > bits_per_period)
242 		return -ENOSPC;
243 
244 	/* Adjust interval as per description */
245 	interval = gcd(interval, periods_in_map);
246 
247 	interval_bits = bits_per_period * interval;
248 	to_reserve = periods_in_map / interval;
249 
250 	/* If start has gotten us past interval then we can't schedule */
251 	if (start >= interval_bits)
252 		return -ENOSPC;
253 
254 	if (only_one_period)
255 		/* Must fit within same period as start; end at begin of next */
256 		first_end = (start / bits_per_period + 1) * bits_per_period;
257 	else
258 		/* Can fit anywhere in the first interval */
259 		first_end = interval_bits;
260 
261 	/*
262 	 * We'll try to pick the first repetition, then see if that time
263 	 * is free for each of the subsequent repetitions.  If it's not
264 	 * we'll adjust the start time for the next search of the first
265 	 * repetition.
266 	 */
267 	while (start + num_bits <= first_end) {
268 		int end;
269 
270 		/* Need to stay within this period */
271 		end = (start / bits_per_period + 1) * bits_per_period;
272 
273 		/* Look for num_bits us in this microframe starting at start */
274 		start = bitmap_find_next_zero_area(map, end, start, num_bits,
275 						   0);
276 
277 		/*
278 		 * We should get start >= end if we fail.  We might be
279 		 * able to check the next microframe depending on the
280 		 * interval, so continue on (start already updated).
281 		 */
282 		if (start >= end) {
283 			start = end;
284 			continue;
285 		}
286 
287 		/* At this point we have a valid point for first one */
288 		for (i = 1; i < to_reserve; i++) {
289 			int ith_start = start + interval_bits * i;
290 			int ith_end = end + interval_bits * i;
291 			int ret;
292 
293 			/* Use this as a dumb "check if bits are 0" */
294 			ret = bitmap_find_next_zero_area(
295 				map, ith_start + num_bits, ith_start, num_bits,
296 				0);
297 
298 			/* We got the right place, continue checking */
299 			if (ret == ith_start)
300 				continue;
301 
302 			/* Move start up for next time and exit for loop */
303 			ith_start = bitmap_find_next_zero_area(
304 				map, ith_end, ith_start, num_bits, 0);
305 			if (ith_start >= ith_end)
306 				/* Need a while new period next time */
307 				start = end;
308 			else
309 				start = ith_start - interval_bits * i;
310 			break;
311 		}
312 
313 		/* If didn't exit the for loop with a break, we have success */
314 		if (i == to_reserve)
315 			break;
316 	}
317 
318 	if (start + num_bits > first_end)
319 		return -ENOSPC;
320 
321 	for (i = 0; i < to_reserve; i++) {
322 		int ith_start = start + interval_bits * i;
323 
324 		bitmap_set(map, ith_start, num_bits);
325 	}
326 
327 	return start;
328 }
329 
330 /**
331  * pmap_unschedule() - Undo work done by pmap_schedule()
332  *
333  * @map:             See pmap_schedule().
334  * @bits_per_period: See pmap_schedule().
335  * @periods_in_map:  See pmap_schedule().
336  * @num_bits:        The number of bits that was passed to schedule.
337  * @interval:        The interval that was passed to schedule.
338  * @start:           The return value from pmap_schedule().
339  */
pmap_unschedule(unsigned long * map,int bits_per_period,int periods_in_map,int num_bits,int interval,int start)340 static void pmap_unschedule(unsigned long *map, int bits_per_period,
341 			    int periods_in_map, int num_bits,
342 			    int interval, int start)
343 {
344 	int interval_bits;
345 	int to_release;
346 	int i;
347 
348 	/* Adjust interval as per description in pmap_schedule() */
349 	interval = gcd(interval, periods_in_map);
350 
351 	interval_bits = bits_per_period * interval;
352 	to_release = periods_in_map / interval;
353 
354 	for (i = 0; i < to_release; i++) {
355 		int ith_start = start + interval_bits * i;
356 
357 		bitmap_clear(map, ith_start, num_bits);
358 	}
359 }
360 
361 /**
362  * dwc2_get_ls_map() - Get the map used for the given qh
363  *
364  * @hsotg: The HCD state structure for the DWC OTG controller.
365  * @qh:    QH for the periodic transfer.
366  *
367  * We'll always get the periodic map out of our TT.  Note that even if we're
368  * running the host straight in low speed / full speed mode it appears as if
369  * a TT is allocated for us, so we'll use it.  If that ever changes we can
370  * add logic here to get a map out of "hsotg" if !qh->do_split.
371  *
372  * Returns: the map or NULL if a map couldn't be found.
373  */
dwc2_get_ls_map(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)374 static unsigned long *dwc2_get_ls_map(struct dwc2_hsotg *hsotg,
375 				      struct dwc2_qh *qh)
376 {
377 	unsigned long *map;
378 
379 	/* Don't expect to be missing a TT and be doing low speed scheduling */
380 	if (WARN_ON(!qh->dwc_tt))
381 		return NULL;
382 
383 	/* Get the map and adjust if this is a multi_tt hub */
384 	map = qh->dwc_tt->periodic_bitmaps;
385 	if (qh->dwc_tt->usb_tt->multi)
386 		map += DWC2_ELEMENTS_PER_LS_BITMAP * (qh->ttport - 1);
387 
388 	return map;
389 }
390 
391 #ifdef DWC2_PRINT_SCHEDULE
392 /*
393  * cat_printf() - A printf() + strcat() helper
394  *
395  * This is useful for concatenating a bunch of strings where each string is
396  * constructed using printf.
397  *
398  * @buf:   The destination buffer; will be updated to point after the printed
399  *         data.
400  * @size:  The number of bytes in the buffer (includes space for '\0').
401  * @fmt:   The format for printf.
402  * @...:   The args for printf.
403  */
404 static __printf(3, 4)
cat_printf(char ** buf,size_t * size,const char * fmt,...)405 void cat_printf(char **buf, size_t *size, const char *fmt, ...)
406 {
407 	va_list args;
408 	int i;
409 
410 	if (*size == 0)
411 		return;
412 
413 	va_start(args, fmt);
414 	i = vsnprintf(*buf, *size, fmt, args);
415 	va_end(args);
416 
417 	if (i >= *size) {
418 		(*buf)[*size - 1] = '\0';
419 		*buf += *size;
420 		*size = 0;
421 	} else {
422 		*buf += i;
423 		*size -= i;
424 	}
425 }
426 
427 /*
428  * pmap_print() - Print the given periodic map
429  *
430  * Will attempt to print out the periodic schedule.
431  *
432  * @map:             See pmap_schedule().
433  * @bits_per_period: See pmap_schedule().
434  * @periods_in_map:  See pmap_schedule().
435  * @period_name:     The name of 1 period, like "uFrame"
436  * @units:           The name of the units, like "us".
437  * @print_fn:        The function to call for printing.
438  * @print_data:      Opaque data to pass to the print function.
439  */
pmap_print(unsigned long * map,int bits_per_period,int periods_in_map,const char * period_name,const char * units,void (* print_fn)(const char * str,void * data),void * print_data)440 static void pmap_print(unsigned long *map, int bits_per_period,
441 		       int periods_in_map, const char *period_name,
442 		       const char *units,
443 		       void (*print_fn)(const char *str, void *data),
444 		       void *print_data)
445 {
446 	int period;
447 
448 	for (period = 0; period < periods_in_map; period++) {
449 		char tmp[64];
450 		char *buf = tmp;
451 		size_t buf_size = sizeof(tmp);
452 		int period_start = period * bits_per_period;
453 		int period_end = period_start + bits_per_period;
454 		int start = 0;
455 		int count = 0;
456 		bool printed = false;
457 		int i;
458 
459 		for (i = period_start; i < period_end + 1; i++) {
460 			/* Handle case when ith bit is set */
461 			if (i < period_end &&
462 			    bitmap_find_next_zero_area(map, i + 1,
463 						       i, 1, 0) != i) {
464 				if (count == 0)
465 					start = i - period_start;
466 				count++;
467 				continue;
468 			}
469 
470 			/* ith bit isn't set; don't care if count == 0 */
471 			if (count == 0)
472 				continue;
473 
474 			if (!printed)
475 				cat_printf(&buf, &buf_size, "%s %d: ",
476 					   period_name, period);
477 			else
478 				cat_printf(&buf, &buf_size, ", ");
479 			printed = true;
480 
481 			cat_printf(&buf, &buf_size, "%d %s -%3d %s", start,
482 				   units, start + count - 1, units);
483 			count = 0;
484 		}
485 
486 		if (printed)
487 			print_fn(tmp, print_data);
488 	}
489 }
490 
491 struct dwc2_qh_print_data {
492 	struct dwc2_hsotg *hsotg;
493 	struct dwc2_qh *qh;
494 };
495 
496 /**
497  * dwc2_qh_print() - Helper function for dwc2_qh_schedule_print()
498  *
499  * @str:  The string to print
500  * @data: A pointer to a struct dwc2_qh_print_data
501  */
dwc2_qh_print(const char * str,void * data)502 static void dwc2_qh_print(const char *str, void *data)
503 {
504 	struct dwc2_qh_print_data *print_data = data;
505 
506 	dwc2_sch_dbg(print_data->hsotg, "QH=%p ...%s\n", print_data->qh, str);
507 }
508 
509 /**
510  * dwc2_qh_schedule_print() - Print the periodic schedule
511  *
512  * @hsotg: The HCD state structure for the DWC OTG controller.
513  * @qh:    QH to print.
514  */
dwc2_qh_schedule_print(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)515 static void dwc2_qh_schedule_print(struct dwc2_hsotg *hsotg,
516 				   struct dwc2_qh *qh)
517 {
518 	struct dwc2_qh_print_data print_data = { hsotg, qh };
519 	int i;
520 
521 	/*
522 	 * The printing functions are quite slow and inefficient.
523 	 * If we don't have tracing turned on, don't run unless the special
524 	 * define is turned on.
525 	 */
526 
527 	if (qh->schedule_low_speed) {
528 		unsigned long *map = dwc2_get_ls_map(hsotg, qh);
529 
530 		dwc2_sch_dbg(hsotg, "QH=%p LS/FS trans: %d=>%d us @ %d us",
531 			     qh, qh->device_us,
532 			     DWC2_ROUND_US_TO_SLICE(qh->device_us),
533 			     DWC2_US_PER_SLICE * qh->ls_start_schedule_slice);
534 
535 		if (map) {
536 			dwc2_sch_dbg(hsotg,
537 				     "QH=%p Whole low/full speed map %p now:\n",
538 				     qh, map);
539 			pmap_print(map, DWC2_LS_PERIODIC_SLICES_PER_FRAME,
540 				   DWC2_LS_SCHEDULE_FRAMES, "Frame ", "slices",
541 				   dwc2_qh_print, &print_data);
542 		}
543 	}
544 
545 	for (i = 0; i < qh->num_hs_transfers; i++) {
546 		struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + i;
547 		int uframe = trans_time->start_schedule_us /
548 			     DWC2_HS_PERIODIC_US_PER_UFRAME;
549 		int rel_us = trans_time->start_schedule_us %
550 			     DWC2_HS_PERIODIC_US_PER_UFRAME;
551 
552 		dwc2_sch_dbg(hsotg,
553 			     "QH=%p HS trans #%d: %d us @ uFrame %d + %d us\n",
554 			     qh, i, trans_time->duration_us, uframe, rel_us);
555 	}
556 	if (qh->num_hs_transfers) {
557 		dwc2_sch_dbg(hsotg, "QH=%p Whole high speed map now:\n", qh);
558 		pmap_print(hsotg->hs_periodic_bitmap,
559 			   DWC2_HS_PERIODIC_US_PER_UFRAME,
560 			   DWC2_HS_SCHEDULE_UFRAMES, "uFrame", "us",
561 			   dwc2_qh_print, &print_data);
562 	}
563 }
564 #else
dwc2_qh_schedule_print(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)565 static inline void dwc2_qh_schedule_print(struct dwc2_hsotg *hsotg,
566 					  struct dwc2_qh *qh) {};
567 #endif
568 
569 /**
570  * dwc2_ls_pmap_schedule() - Schedule a low speed QH
571  *
572  * @hsotg:        The HCD state structure for the DWC OTG controller.
573  * @qh:           QH for the periodic transfer.
574  * @search_slice: We'll start trying to schedule at the passed slice.
575  *                Remember that slices are the units of the low speed
576  *                schedule (think 25us or so).
577  *
578  * Wraps pmap_schedule() with the right parameters for low speed scheduling.
579  *
580  * Normally we schedule low speed devices on the map associated with the TT.
581  *
582  * Returns: 0 for success or an error code.
583  */
dwc2_ls_pmap_schedule(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,int search_slice)584 static int dwc2_ls_pmap_schedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
585 				 int search_slice)
586 {
587 	int slices = DIV_ROUND_UP(qh->device_us, DWC2_US_PER_SLICE);
588 	unsigned long *map = dwc2_get_ls_map(hsotg, qh);
589 	int slice;
590 
591 	if (!map)
592 		return -EINVAL;
593 
594 	/*
595 	 * Schedule on the proper low speed map with our low speed scheduling
596 	 * parameters.  Note that we use the "device_interval" here since
597 	 * we want the low speed interval and the only way we'd be in this
598 	 * function is if the device is low speed.
599 	 *
600 	 * If we happen to be doing low speed and high speed scheduling for the
601 	 * same transaction (AKA we have a split) we always do low speed first.
602 	 * That means we can always pass "false" for only_one_period (that
603 	 * parameters is only useful when we're trying to get one schedule to
604 	 * match what we already planned in the other schedule).
605 	 */
606 	slice = pmap_schedule(map, DWC2_LS_PERIODIC_SLICES_PER_FRAME,
607 			      DWC2_LS_SCHEDULE_FRAMES, slices,
608 			      qh->device_interval, search_slice, false);
609 
610 	if (slice < 0)
611 		return slice;
612 
613 	qh->ls_start_schedule_slice = slice;
614 	return 0;
615 }
616 
617 /**
618  * dwc2_ls_pmap_unschedule() - Undo work done by dwc2_ls_pmap_schedule()
619  *
620  * @hsotg:       The HCD state structure for the DWC OTG controller.
621  * @qh:          QH for the periodic transfer.
622  */
dwc2_ls_pmap_unschedule(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)623 static void dwc2_ls_pmap_unschedule(struct dwc2_hsotg *hsotg,
624 				    struct dwc2_qh *qh)
625 {
626 	int slices = DIV_ROUND_UP(qh->device_us, DWC2_US_PER_SLICE);
627 	unsigned long *map = dwc2_get_ls_map(hsotg, qh);
628 
629 	/* Schedule should have failed, so no worries about no error code */
630 	if (!map)
631 		return;
632 
633 	pmap_unschedule(map, DWC2_LS_PERIODIC_SLICES_PER_FRAME,
634 			DWC2_LS_SCHEDULE_FRAMES, slices, qh->device_interval,
635 			qh->ls_start_schedule_slice);
636 }
637 
638 /**
639  * dwc2_hs_pmap_schedule - Schedule in the main high speed schedule
640  *
641  * This will schedule something on the main dwc2 schedule.
642  *
643  * We'll start looking in qh->hs_transfers[index].start_schedule_us.  We'll
644  * update this with the result upon success.  We also use the duration from
645  * the same structure.
646  *
647  * @hsotg:           The HCD state structure for the DWC OTG controller.
648  * @qh:              QH for the periodic transfer.
649  * @only_one_period: If true we will limit ourselves to just looking at
650  *                   one period (aka one 100us chunk).  This is used if we have
651  *                   already scheduled something on the low speed schedule and
652  *                   need to find something that matches on the high speed one.
653  * @index:           The index into qh->hs_transfers that we're working with.
654  *
655  * Returns: 0 for success or an error code.  Upon success the
656  *          dwc2_hs_transfer_time specified by "index" will be updated.
657  */
dwc2_hs_pmap_schedule(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,bool only_one_period,int index)658 static int dwc2_hs_pmap_schedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
659 				 bool only_one_period, int index)
660 {
661 	struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + index;
662 	int us;
663 
664 	us = pmap_schedule(hsotg->hs_periodic_bitmap,
665 			   DWC2_HS_PERIODIC_US_PER_UFRAME,
666 			   DWC2_HS_SCHEDULE_UFRAMES, trans_time->duration_us,
667 			   qh->host_interval, trans_time->start_schedule_us,
668 			   only_one_period);
669 
670 	if (us < 0)
671 		return us;
672 
673 	trans_time->start_schedule_us = us;
674 	return 0;
675 }
676 
677 /**
678  * dwc2_hs_pmap_unschedule() - Undo work done by dwc2_hs_pmap_schedule()
679  *
680  * @hsotg:       The HCD state structure for the DWC OTG controller.
681  * @qh:          QH for the periodic transfer.
682  * @index:       Transfer index
683  */
dwc2_hs_pmap_unschedule(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,int index)684 static void dwc2_hs_pmap_unschedule(struct dwc2_hsotg *hsotg,
685 				    struct dwc2_qh *qh, int index)
686 {
687 	struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + index;
688 
689 	pmap_unschedule(hsotg->hs_periodic_bitmap,
690 			DWC2_HS_PERIODIC_US_PER_UFRAME,
691 			DWC2_HS_SCHEDULE_UFRAMES, trans_time->duration_us,
692 			qh->host_interval, trans_time->start_schedule_us);
693 }
694 
695 /**
696  * dwc2_uframe_schedule_split - Schedule a QH for a periodic split xfer.
697  *
698  * This is the most complicated thing in USB.  We have to find matching time
699  * in both the global high speed schedule for the port and the low speed
700  * schedule for the TT associated with the given device.
701  *
702  * Being here means that the host must be running in high speed mode and the
703  * device is in low or full speed mode (and behind a hub).
704  *
705  * @hsotg:       The HCD state structure for the DWC OTG controller.
706  * @qh:          QH for the periodic transfer.
707  */
dwc2_uframe_schedule_split(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)708 static int dwc2_uframe_schedule_split(struct dwc2_hsotg *hsotg,
709 				      struct dwc2_qh *qh)
710 {
711 	int bytecount = qh->maxp_mult * qh->maxp;
712 	int ls_search_slice;
713 	int err = 0;
714 	int host_interval_in_sched;
715 
716 	/*
717 	 * The interval (how often to repeat) in the actual host schedule.
718 	 * See pmap_schedule() for gcd() explanation.
719 	 */
720 	host_interval_in_sched = gcd(qh->host_interval,
721 				     DWC2_HS_SCHEDULE_UFRAMES);
722 
723 	/*
724 	 * We always try to find space in the low speed schedule first, then
725 	 * try to find high speed time that matches.  If we don't, we'll bump
726 	 * up the place we start searching in the low speed schedule and try
727 	 * again.  To start we'll look right at the beginning of the low speed
728 	 * schedule.
729 	 *
730 	 * Note that this will tend to front-load the high speed schedule.
731 	 * We may eventually want to try to avoid this by either considering
732 	 * both schedules together or doing some sort of round robin.
733 	 */
734 	ls_search_slice = 0;
735 
736 	while (ls_search_slice < DWC2_LS_SCHEDULE_SLICES) {
737 		int start_s_uframe;
738 		int ssplit_s_uframe;
739 		int second_s_uframe;
740 		int rel_uframe;
741 		int first_count;
742 		int middle_count;
743 		int end_count;
744 		int first_data_bytes;
745 		int other_data_bytes;
746 		int i;
747 
748 		if (qh->schedule_low_speed) {
749 			err = dwc2_ls_pmap_schedule(hsotg, qh, ls_search_slice);
750 
751 			/*
752 			 * If we got an error here there's no other magic we
753 			 * can do, so bail.  All the looping above is only
754 			 * helpful to redo things if we got a low speed slot
755 			 * and then couldn't find a matching high speed slot.
756 			 */
757 			if (err)
758 				return err;
759 		} else {
760 			/* Must be missing the tt structure?  Why? */
761 			WARN_ON_ONCE(1);
762 		}
763 
764 		/*
765 		 * This will give us a number 0 - 7 if
766 		 * DWC2_LS_SCHEDULE_FRAMES == 1, or 0 - 15 if == 2, or ...
767 		 */
768 		start_s_uframe = qh->ls_start_schedule_slice /
769 				 DWC2_SLICES_PER_UFRAME;
770 
771 		/* Get a number that's always 0 - 7 */
772 		rel_uframe = (start_s_uframe % 8);
773 
774 		/*
775 		 * If we were going to start in uframe 7 then we would need to
776 		 * issue a start split in uframe 6, which spec says is not OK.
777 		 * Move on to the next full frame (assuming there is one).
778 		 *
779 		 * See 11.18.4 Host Split Transaction Scheduling Requirements
780 		 * bullet 1.
781 		 */
782 		if (rel_uframe == 7) {
783 			if (qh->schedule_low_speed)
784 				dwc2_ls_pmap_unschedule(hsotg, qh);
785 			ls_search_slice =
786 				(qh->ls_start_schedule_slice /
787 				 DWC2_LS_PERIODIC_SLICES_PER_FRAME + 1) *
788 				DWC2_LS_PERIODIC_SLICES_PER_FRAME;
789 			continue;
790 		}
791 
792 		/*
793 		 * For ISOC in:
794 		 * - start split            (frame -1)
795 		 * - complete split w/ data (frame +1)
796 		 * - complete split w/ data (frame +2)
797 		 * - ...
798 		 * - complete split w/ data (frame +num_data_packets)
799 		 * - complete split w/ data (frame +num_data_packets+1)
800 		 * - complete split w/ data (frame +num_data_packets+2, max 8)
801 		 *   ...though if frame was "0" then max is 7...
802 		 *
803 		 * For ISOC out we might need to do:
804 		 * - start split w/ data    (frame -1)
805 		 * - start split w/ data    (frame +0)
806 		 * - ...
807 		 * - start split w/ data    (frame +num_data_packets-2)
808 		 *
809 		 * For INTERRUPT in we might need to do:
810 		 * - start split            (frame -1)
811 		 * - complete split w/ data (frame +1)
812 		 * - complete split w/ data (frame +2)
813 		 * - complete split w/ data (frame +3, max 8)
814 		 *
815 		 * For INTERRUPT out we might need to do:
816 		 * - start split w/ data    (frame -1)
817 		 * - complete split         (frame +1)
818 		 * - complete split         (frame +2)
819 		 * - complete split         (frame +3, max 8)
820 		 *
821 		 * Start adjusting!
822 		 */
823 		ssplit_s_uframe = (start_s_uframe +
824 				   host_interval_in_sched - 1) %
825 				  host_interval_in_sched;
826 		if (qh->ep_type == USB_ENDPOINT_XFER_ISOC && !qh->ep_is_in)
827 			second_s_uframe = start_s_uframe;
828 		else
829 			second_s_uframe = start_s_uframe + 1;
830 
831 		/* First data transfer might not be all 188 bytes. */
832 		first_data_bytes = 188 -
833 			DIV_ROUND_UP(188 * (qh->ls_start_schedule_slice %
834 					    DWC2_SLICES_PER_UFRAME),
835 				     DWC2_SLICES_PER_UFRAME);
836 		if (first_data_bytes > bytecount)
837 			first_data_bytes = bytecount;
838 		other_data_bytes = bytecount - first_data_bytes;
839 
840 		/*
841 		 * For now, skip OUT xfers where first xfer is partial
842 		 *
843 		 * Main dwc2 code assumes:
844 		 * - INT transfers never get split in two.
845 		 * - ISOC transfers can always transfer 188 bytes the first
846 		 *   time.
847 		 *
848 		 * Until that code is fixed, try again if the first transfer
849 		 * couldn't transfer everything.
850 		 *
851 		 * This code can be removed if/when the rest of dwc2 handles
852 		 * the above cases.  Until it's fixed we just won't be able
853 		 * to schedule quite as tightly.
854 		 */
855 		if (!qh->ep_is_in &&
856 		    (first_data_bytes != min_t(int, 188, bytecount))) {
857 			dwc2_sch_dbg(hsotg,
858 				     "QH=%p avoiding broken 1st xfer (%d, %d)\n",
859 				     qh, first_data_bytes, bytecount);
860 			if (qh->schedule_low_speed)
861 				dwc2_ls_pmap_unschedule(hsotg, qh);
862 			ls_search_slice = (start_s_uframe + 1) *
863 				DWC2_SLICES_PER_UFRAME;
864 			continue;
865 		}
866 
867 		/* Start by assuming transfers for the bytes */
868 		qh->num_hs_transfers = 1 + DIV_ROUND_UP(other_data_bytes, 188);
869 
870 		/*
871 		 * Everything except ISOC OUT has extra transfers.  Rules are
872 		 * complicated.  See 11.18.4 Host Split Transaction Scheduling
873 		 * Requirements bullet 3.
874 		 */
875 		if (qh->ep_type == USB_ENDPOINT_XFER_INT) {
876 			if (rel_uframe == 6)
877 				qh->num_hs_transfers += 2;
878 			else
879 				qh->num_hs_transfers += 3;
880 
881 			if (qh->ep_is_in) {
882 				/*
883 				 * First is start split, middle/end is data.
884 				 * Allocate full data bytes for all data.
885 				 */
886 				first_count = 4;
887 				middle_count = bytecount;
888 				end_count = bytecount;
889 			} else {
890 				/*
891 				 * First is data, middle/end is complete.
892 				 * First transfer and second can have data.
893 				 * Rest should just have complete split.
894 				 */
895 				first_count = first_data_bytes;
896 				middle_count = max_t(int, 4, other_data_bytes);
897 				end_count = 4;
898 			}
899 		} else {
900 			if (qh->ep_is_in) {
901 				int last;
902 
903 				/* Account for the start split */
904 				qh->num_hs_transfers++;
905 
906 				/* Calculate "L" value from spec */
907 				last = rel_uframe + qh->num_hs_transfers + 1;
908 
909 				/* Start with basic case */
910 				if (last <= 6)
911 					qh->num_hs_transfers += 2;
912 				else
913 					qh->num_hs_transfers += 1;
914 
915 				/* Adjust downwards */
916 				if (last >= 6 && rel_uframe == 0)
917 					qh->num_hs_transfers--;
918 
919 				/* 1st = start; rest can contain data */
920 				first_count = 4;
921 				middle_count = min_t(int, 188, bytecount);
922 				end_count = middle_count;
923 			} else {
924 				/* All contain data, last might be smaller */
925 				first_count = first_data_bytes;
926 				middle_count = min_t(int, 188,
927 						     other_data_bytes);
928 				end_count = other_data_bytes % 188;
929 			}
930 		}
931 
932 		/* Assign durations per uFrame */
933 		qh->hs_transfers[0].duration_us = HS_USECS_ISO(first_count);
934 		for (i = 1; i < qh->num_hs_transfers - 1; i++)
935 			qh->hs_transfers[i].duration_us =
936 				HS_USECS_ISO(middle_count);
937 		if (qh->num_hs_transfers > 1)
938 			qh->hs_transfers[qh->num_hs_transfers - 1].duration_us =
939 				HS_USECS_ISO(end_count);
940 
941 		/*
942 		 * Assign start us.  The call below to dwc2_hs_pmap_schedule()
943 		 * will start with these numbers but may adjust within the same
944 		 * microframe.
945 		 */
946 		qh->hs_transfers[0].start_schedule_us =
947 			ssplit_s_uframe * DWC2_HS_PERIODIC_US_PER_UFRAME;
948 		for (i = 1; i < qh->num_hs_transfers; i++)
949 			qh->hs_transfers[i].start_schedule_us =
950 				((second_s_uframe + i - 1) %
951 				 DWC2_HS_SCHEDULE_UFRAMES) *
952 				DWC2_HS_PERIODIC_US_PER_UFRAME;
953 
954 		/* Try to schedule with filled in hs_transfers above */
955 		for (i = 0; i < qh->num_hs_transfers; i++) {
956 			err = dwc2_hs_pmap_schedule(hsotg, qh, true, i);
957 			if (err)
958 				break;
959 		}
960 
961 		/* If we scheduled all w/out breaking out then we're all good */
962 		if (i == qh->num_hs_transfers)
963 			break;
964 
965 		for (; i >= 0; i--)
966 			dwc2_hs_pmap_unschedule(hsotg, qh, i);
967 
968 		if (qh->schedule_low_speed)
969 			dwc2_ls_pmap_unschedule(hsotg, qh);
970 
971 		/* Try again starting in the next microframe */
972 		ls_search_slice = (start_s_uframe + 1) * DWC2_SLICES_PER_UFRAME;
973 	}
974 
975 	if (ls_search_slice >= DWC2_LS_SCHEDULE_SLICES)
976 		return -ENOSPC;
977 
978 	return 0;
979 }
980 
981 /**
982  * dwc2_uframe_schedule_hs - Schedule a QH for a periodic high speed xfer.
983  *
984  * Basically this just wraps dwc2_hs_pmap_schedule() to provide a clean
985  * interface.
986  *
987  * @hsotg:       The HCD state structure for the DWC OTG controller.
988  * @qh:          QH for the periodic transfer.
989  */
dwc2_uframe_schedule_hs(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)990 static int dwc2_uframe_schedule_hs(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
991 {
992 	/* In non-split host and device time are the same */
993 	WARN_ON(qh->host_us != qh->device_us);
994 	WARN_ON(qh->host_interval != qh->device_interval);
995 	WARN_ON(qh->num_hs_transfers != 1);
996 
997 	/* We'll have one transfer; init start to 0 before calling scheduler */
998 	qh->hs_transfers[0].start_schedule_us = 0;
999 	qh->hs_transfers[0].duration_us = qh->host_us;
1000 
1001 	return dwc2_hs_pmap_schedule(hsotg, qh, false, 0);
1002 }
1003 
1004 /**
1005  * dwc2_uframe_schedule_ls - Schedule a QH for a periodic low/full speed xfer.
1006  *
1007  * Basically this just wraps dwc2_ls_pmap_schedule() to provide a clean
1008  * interface.
1009  *
1010  * @hsotg:       The HCD state structure for the DWC OTG controller.
1011  * @qh:          QH for the periodic transfer.
1012  */
dwc2_uframe_schedule_ls(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1013 static int dwc2_uframe_schedule_ls(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1014 {
1015 	/* In non-split host and device time are the same */
1016 	WARN_ON(qh->host_us != qh->device_us);
1017 	WARN_ON(qh->host_interval != qh->device_interval);
1018 	WARN_ON(!qh->schedule_low_speed);
1019 
1020 	/* Run on the main low speed schedule (no split = no hub = no TT) */
1021 	return dwc2_ls_pmap_schedule(hsotg, qh, 0);
1022 }
1023 
1024 /**
1025  * dwc2_uframe_schedule - Schedule a QH for a periodic xfer.
1026  *
1027  * Calls one of the 3 sub-function depending on what type of transfer this QH
1028  * is for.  Also adds some printing.
1029  *
1030  * @hsotg:       The HCD state structure for the DWC OTG controller.
1031  * @qh:          QH for the periodic transfer.
1032  */
dwc2_uframe_schedule(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1033 static int dwc2_uframe_schedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1034 {
1035 	int ret;
1036 
1037 	if (qh->dev_speed == USB_SPEED_HIGH)
1038 		ret = dwc2_uframe_schedule_hs(hsotg, qh);
1039 	else if (!qh->do_split)
1040 		ret = dwc2_uframe_schedule_ls(hsotg, qh);
1041 	else
1042 		ret = dwc2_uframe_schedule_split(hsotg, qh);
1043 
1044 	if (ret)
1045 		dwc2_sch_dbg(hsotg, "QH=%p Failed to schedule %d\n", qh, ret);
1046 	else
1047 		dwc2_qh_schedule_print(hsotg, qh);
1048 
1049 	return ret;
1050 }
1051 
1052 /**
1053  * dwc2_uframe_unschedule - Undoes dwc2_uframe_schedule().
1054  *
1055  * @hsotg:       The HCD state structure for the DWC OTG controller.
1056  * @qh:          QH for the periodic transfer.
1057  */
dwc2_uframe_unschedule(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1058 static void dwc2_uframe_unschedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1059 {
1060 	int i;
1061 
1062 	for (i = 0; i < qh->num_hs_transfers; i++)
1063 		dwc2_hs_pmap_unschedule(hsotg, qh, i);
1064 
1065 	if (qh->schedule_low_speed)
1066 		dwc2_ls_pmap_unschedule(hsotg, qh);
1067 
1068 	dwc2_sch_dbg(hsotg, "QH=%p Unscheduled\n", qh);
1069 }
1070 
1071 /**
1072  * dwc2_pick_first_frame() - Choose 1st frame for qh that's already scheduled
1073  *
1074  * Takes a qh that has already been scheduled (which means we know we have the
1075  * bandwdith reserved for us) and set the next_active_frame and the
1076  * start_active_frame.
1077  *
1078  * This is expected to be called on qh's that weren't previously actively
1079  * running.  It just picks the next frame that we can fit into without any
1080  * thought about the past.
1081  *
1082  * @hsotg: The HCD state structure for the DWC OTG controller
1083  * @qh:    QH for a periodic endpoint
1084  *
1085  */
dwc2_pick_first_frame(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1086 static void dwc2_pick_first_frame(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1087 {
1088 	u16 frame_number;
1089 	u16 earliest_frame;
1090 	u16 next_active_frame;
1091 	u16 relative_frame;
1092 	u16 interval;
1093 
1094 	/*
1095 	 * Use the real frame number rather than the cached value as of the
1096 	 * last SOF to give us a little extra slop.
1097 	 */
1098 	frame_number = dwc2_hcd_get_frame_number(hsotg);
1099 
1100 	/*
1101 	 * We wouldn't want to start any earlier than the next frame just in
1102 	 * case the frame number ticks as we're doing this calculation.
1103 	 *
1104 	 * NOTE: if we could quantify how long till we actually get scheduled
1105 	 * we might be able to avoid the "+ 1" by looking at the upper part of
1106 	 * HFNUM (the FRREM field).  For now we'll just use the + 1 though.
1107 	 */
1108 	earliest_frame = dwc2_frame_num_inc(frame_number, 1);
1109 	next_active_frame = earliest_frame;
1110 
1111 	/* Get the "no microframe schduler" out of the way... */
1112 	if (!hsotg->params.uframe_sched) {
1113 		if (qh->do_split)
1114 			/* Splits are active at microframe 0 minus 1 */
1115 			next_active_frame |= 0x7;
1116 		goto exit;
1117 	}
1118 
1119 	if (qh->dev_speed == USB_SPEED_HIGH || qh->do_split) {
1120 		/*
1121 		 * We're either at high speed or we're doing a split (which
1122 		 * means we're talking high speed to a hub).  In any case
1123 		 * the first frame should be based on when the first scheduled
1124 		 * event is.
1125 		 */
1126 		WARN_ON(qh->num_hs_transfers < 1);
1127 
1128 		relative_frame = qh->hs_transfers[0].start_schedule_us /
1129 				 DWC2_HS_PERIODIC_US_PER_UFRAME;
1130 
1131 		/* Adjust interval as per high speed schedule */
1132 		interval = gcd(qh->host_interval, DWC2_HS_SCHEDULE_UFRAMES);
1133 
1134 	} else {
1135 		/*
1136 		 * Low or full speed directly on dwc2.  Just about the same
1137 		 * as high speed but on a different schedule and with slightly
1138 		 * different adjustments.  Note that this works because when
1139 		 * the host and device are both low speed then frames in the
1140 		 * controller tick at low speed.
1141 		 */
1142 		relative_frame = qh->ls_start_schedule_slice /
1143 				 DWC2_LS_PERIODIC_SLICES_PER_FRAME;
1144 		interval = gcd(qh->host_interval, DWC2_LS_SCHEDULE_FRAMES);
1145 	}
1146 
1147 	/* Scheduler messed up if frame is past interval */
1148 	WARN_ON(relative_frame >= interval);
1149 
1150 	/*
1151 	 * We know interval must divide (HFNUM_MAX_FRNUM + 1) now that we've
1152 	 * done the gcd(), so it's safe to move to the beginning of the current
1153 	 * interval like this.
1154 	 *
1155 	 * After this we might be before earliest_frame, but don't worry,
1156 	 * we'll fix it...
1157 	 */
1158 	next_active_frame = (next_active_frame / interval) * interval;
1159 
1160 	/*
1161 	 * Actually choose to start at the frame number we've been
1162 	 * scheduled for.
1163 	 */
1164 	next_active_frame = dwc2_frame_num_inc(next_active_frame,
1165 					       relative_frame);
1166 
1167 	/*
1168 	 * We actually need 1 frame before since the next_active_frame is
1169 	 * the frame number we'll be put on the ready list and we won't be on
1170 	 * the bus until 1 frame later.
1171 	 */
1172 	next_active_frame = dwc2_frame_num_dec(next_active_frame, 1);
1173 
1174 	/*
1175 	 * By now we might actually be before the earliest_frame.  Let's move
1176 	 * up intervals until we're not.
1177 	 */
1178 	while (dwc2_frame_num_gt(earliest_frame, next_active_frame))
1179 		next_active_frame = dwc2_frame_num_inc(next_active_frame,
1180 						       interval);
1181 
1182 exit:
1183 	qh->next_active_frame = next_active_frame;
1184 	qh->start_active_frame = next_active_frame;
1185 
1186 	dwc2_sch_vdbg(hsotg, "QH=%p First fn=%04x nxt=%04x\n",
1187 		      qh, frame_number, qh->next_active_frame);
1188 }
1189 
1190 /**
1191  * dwc2_do_reserve() - Make a periodic reservation
1192  *
1193  * Try to allocate space in the periodic schedule.  Depending on parameters
1194  * this might use the microframe scheduler or the dumb scheduler.
1195  *
1196  * @hsotg: The HCD state structure for the DWC OTG controller
1197  * @qh:    QH for the periodic transfer.
1198  *
1199  * Returns: 0 upon success; error upon failure.
1200  */
dwc2_do_reserve(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1201 static int dwc2_do_reserve(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1202 {
1203 	int status;
1204 
1205 	if (hsotg->params.uframe_sched) {
1206 		status = dwc2_uframe_schedule(hsotg, qh);
1207 	} else {
1208 		status = dwc2_periodic_channel_available(hsotg);
1209 		if (status) {
1210 			dev_info(hsotg->dev,
1211 				 "%s: No host channel available for periodic transfer\n",
1212 				 __func__);
1213 			return status;
1214 		}
1215 
1216 		status = dwc2_check_periodic_bandwidth(hsotg, qh);
1217 	}
1218 
1219 	if (status) {
1220 		dev_dbg(hsotg->dev,
1221 			"%s: Insufficient periodic bandwidth for periodic transfer\n",
1222 			__func__);
1223 		return status;
1224 	}
1225 
1226 	if (!hsotg->params.uframe_sched)
1227 		/* Reserve periodic channel */
1228 		hsotg->periodic_channels++;
1229 
1230 	/* Update claimed usecs per (micro)frame */
1231 	hsotg->periodic_usecs += qh->host_us;
1232 
1233 	dwc2_pick_first_frame(hsotg, qh);
1234 
1235 	return 0;
1236 }
1237 
1238 /**
1239  * dwc2_do_unreserve() - Actually release the periodic reservation
1240  *
1241  * This function actually releases the periodic bandwidth that was reserved
1242  * by the given qh.
1243  *
1244  * @hsotg: The HCD state structure for the DWC OTG controller
1245  * @qh:    QH for the periodic transfer.
1246  */
dwc2_do_unreserve(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1247 static void dwc2_do_unreserve(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1248 {
1249 	assert_spin_locked(&hsotg->lock);
1250 
1251 	WARN_ON(!qh->unreserve_pending);
1252 
1253 	/* No more unreserve pending--we're doing it */
1254 	qh->unreserve_pending = false;
1255 
1256 	if (WARN_ON(!list_empty(&qh->qh_list_entry)))
1257 		list_del_init(&qh->qh_list_entry);
1258 
1259 	/* Update claimed usecs per (micro)frame */
1260 	hsotg->periodic_usecs -= qh->host_us;
1261 
1262 	if (hsotg->params.uframe_sched) {
1263 		dwc2_uframe_unschedule(hsotg, qh);
1264 	} else {
1265 		/* Release periodic channel reservation */
1266 		hsotg->periodic_channels--;
1267 	}
1268 }
1269 
1270 /**
1271  * dwc2_unreserve_timer_fn() - Timer function to release periodic reservation
1272  *
1273  * According to the kernel doc for usb_submit_urb() (specifically the part about
1274  * "Reserved Bandwidth Transfers"), we need to keep a reservation active as
1275  * long as a device driver keeps submitting.  Since we're using HCD_BH to give
1276  * back the URB we need to give the driver a little bit of time before we
1277  * release the reservation.  This worker is called after the appropriate
1278  * delay.
1279  *
1280  * @t: Address to a qh unreserve_work.
1281  */
dwc2_unreserve_timer_fn(struct timer_list * t)1282 static void dwc2_unreserve_timer_fn(struct timer_list *t)
1283 {
1284 	struct dwc2_qh *qh = from_timer(qh, t, unreserve_timer);
1285 	struct dwc2_hsotg *hsotg = qh->hsotg;
1286 	unsigned long flags;
1287 
1288 	/*
1289 	 * Wait for the lock, or for us to be scheduled again.  We
1290 	 * could be scheduled again if:
1291 	 * - We started executing but didn't get the lock yet.
1292 	 * - A new reservation came in, but cancel didn't take effect
1293 	 *   because we already started executing.
1294 	 * - The timer has been kicked again.
1295 	 * In that case cancel and wait for the next call.
1296 	 */
1297 	while (!spin_trylock_irqsave(&hsotg->lock, flags)) {
1298 		if (timer_pending(&qh->unreserve_timer))
1299 			return;
1300 	}
1301 
1302 	/*
1303 	 * Might be no more unreserve pending if:
1304 	 * - We started executing but didn't get the lock yet.
1305 	 * - A new reservation came in, but cancel didn't take effect
1306 	 *   because we already started executing.
1307 	 *
1308 	 * We can't put this in the loop above because unreserve_pending needs
1309 	 * to be accessed under lock, so we can only check it once we got the
1310 	 * lock.
1311 	 */
1312 	if (qh->unreserve_pending)
1313 		dwc2_do_unreserve(hsotg, qh);
1314 
1315 	spin_unlock_irqrestore(&hsotg->lock, flags);
1316 }
1317 
1318 /**
1319  * dwc2_check_max_xfer_size() - Checks that the max transfer size allowed in a
1320  * host channel is large enough to handle the maximum data transfer in a single
1321  * (micro)frame for a periodic transfer
1322  *
1323  * @hsotg: The HCD state structure for the DWC OTG controller
1324  * @qh:    QH for a periodic endpoint
1325  *
1326  * Return: 0 if successful, negative error code otherwise
1327  */
dwc2_check_max_xfer_size(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1328 static int dwc2_check_max_xfer_size(struct dwc2_hsotg *hsotg,
1329 				    struct dwc2_qh *qh)
1330 {
1331 	u32 max_xfer_size;
1332 	u32 max_channel_xfer_size;
1333 	int status = 0;
1334 
1335 	max_xfer_size = qh->maxp * qh->maxp_mult;
1336 	max_channel_xfer_size = hsotg->params.max_transfer_size;
1337 
1338 	if (max_xfer_size > max_channel_xfer_size) {
1339 		dev_err(hsotg->dev,
1340 			"%s: Periodic xfer length %d > max xfer length for channel %d\n",
1341 			__func__, max_xfer_size, max_channel_xfer_size);
1342 		status = -ENOSPC;
1343 	}
1344 
1345 	return status;
1346 }
1347 
1348 /**
1349  * dwc2_schedule_periodic() - Schedules an interrupt or isochronous transfer in
1350  * the periodic schedule
1351  *
1352  * @hsotg: The HCD state structure for the DWC OTG controller
1353  * @qh:    QH for the periodic transfer. The QH should already contain the
1354  *         scheduling information.
1355  *
1356  * Return: 0 if successful, negative error code otherwise
1357  */
dwc2_schedule_periodic(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1358 static int dwc2_schedule_periodic(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1359 {
1360 	int status;
1361 
1362 	status = dwc2_check_max_xfer_size(hsotg, qh);
1363 	if (status) {
1364 		dev_dbg(hsotg->dev,
1365 			"%s: Channel max transfer size too small for periodic transfer\n",
1366 			__func__);
1367 		return status;
1368 	}
1369 
1370 	/* Cancel pending unreserve; if canceled OK, unreserve was pending */
1371 	if (del_timer(&qh->unreserve_timer))
1372 		WARN_ON(!qh->unreserve_pending);
1373 
1374 	/*
1375 	 * Only need to reserve if there's not an unreserve pending, since if an
1376 	 * unreserve is pending then by definition our old reservation is still
1377 	 * valid.  Unreserve might still be pending even if we didn't cancel if
1378 	 * dwc2_unreserve_timer_fn() already started.  Code in the timer handles
1379 	 * that case.
1380 	 */
1381 	if (!qh->unreserve_pending) {
1382 		status = dwc2_do_reserve(hsotg, qh);
1383 		if (status)
1384 			return status;
1385 	} else {
1386 		/*
1387 		 * It might have been a while, so make sure that frame_number
1388 		 * is still good.  Note: we could also try to use the similar
1389 		 * dwc2_next_periodic_start() but that schedules much more
1390 		 * tightly and we might need to hurry and queue things up.
1391 		 */
1392 		if (dwc2_frame_num_le(qh->next_active_frame,
1393 				      hsotg->frame_number))
1394 			dwc2_pick_first_frame(hsotg, qh);
1395 	}
1396 
1397 	qh->unreserve_pending = 0;
1398 
1399 	if (hsotg->params.dma_desc_enable)
1400 		/* Don't rely on SOF and start in ready schedule */
1401 		list_add_tail(&qh->qh_list_entry, &hsotg->periodic_sched_ready);
1402 	else
1403 		/* Always start in inactive schedule */
1404 		list_add_tail(&qh->qh_list_entry,
1405 			      &hsotg->periodic_sched_inactive);
1406 
1407 	return 0;
1408 }
1409 
1410 /**
1411  * dwc2_deschedule_periodic() - Removes an interrupt or isochronous transfer
1412  * from the periodic schedule
1413  *
1414  * @hsotg: The HCD state structure for the DWC OTG controller
1415  * @qh:	   QH for the periodic transfer
1416  */
dwc2_deschedule_periodic(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1417 static void dwc2_deschedule_periodic(struct dwc2_hsotg *hsotg,
1418 				     struct dwc2_qh *qh)
1419 {
1420 	bool did_modify;
1421 
1422 	assert_spin_locked(&hsotg->lock);
1423 
1424 	/*
1425 	 * Schedule the unreserve to happen in a little bit.  Cases here:
1426 	 * - Unreserve worker might be sitting there waiting to grab the lock.
1427 	 *   In this case it will notice it's been schedule again and will
1428 	 *   quit.
1429 	 * - Unreserve worker might not be scheduled.
1430 	 *
1431 	 * We should never already be scheduled since dwc2_schedule_periodic()
1432 	 * should have canceled the scheduled unreserve timer (hence the
1433 	 * warning on did_modify).
1434 	 *
1435 	 * We add + 1 to the timer to guarantee that at least 1 jiffy has
1436 	 * passed (otherwise if the jiffy counter might tick right after we
1437 	 * read it and we'll get no delay).
1438 	 */
1439 	did_modify = mod_timer(&qh->unreserve_timer,
1440 			       jiffies + DWC2_UNRESERVE_DELAY + 1);
1441 	WARN_ON(did_modify);
1442 	qh->unreserve_pending = 1;
1443 
1444 	list_del_init(&qh->qh_list_entry);
1445 }
1446 
1447 /**
1448  * dwc2_wait_timer_fn() - Timer function to re-queue after waiting
1449  *
1450  * As per the spec, a NAK indicates that "a function is temporarily unable to
1451  * transmit or receive data, but will eventually be able to do so without need
1452  * of host intervention".
1453  *
1454  * That means that when we encounter a NAK we're supposed to retry.
1455  *
1456  * ...but if we retry right away (from the interrupt handler that saw the NAK)
1457  * then we can end up with an interrupt storm (if the other side keeps NAKing
1458  * us) because on slow enough CPUs it could take us longer to get out of the
1459  * interrupt routine than it takes for the device to send another NAK.  That
1460  * leads to a constant stream of NAK interrupts and the CPU locks.
1461  *
1462  * ...so instead of retrying right away in the case of a NAK we'll set a timer
1463  * to retry some time later.  This function handles that timer and moves the
1464  * qh back to the "inactive" list, then queues transactions.
1465  *
1466  * @t: Pointer to wait_timer in a qh.
1467  *
1468  * Return: HRTIMER_NORESTART to not automatically restart this timer.
1469  */
dwc2_wait_timer_fn(struct hrtimer * t)1470 static enum hrtimer_restart dwc2_wait_timer_fn(struct hrtimer *t)
1471 {
1472 	struct dwc2_qh *qh = container_of(t, struct dwc2_qh, wait_timer);
1473 	struct dwc2_hsotg *hsotg = qh->hsotg;
1474 	unsigned long flags;
1475 
1476 	spin_lock_irqsave(&hsotg->lock, flags);
1477 
1478 	/*
1479 	 * We'll set wait_timer_cancel to true if we want to cancel this
1480 	 * operation in dwc2_hcd_qh_unlink().
1481 	 */
1482 	if (!qh->wait_timer_cancel) {
1483 		enum dwc2_transaction_type tr_type;
1484 
1485 		qh->want_wait = false;
1486 
1487 		list_move(&qh->qh_list_entry,
1488 			  &hsotg->non_periodic_sched_inactive);
1489 
1490 		tr_type = dwc2_hcd_select_transactions(hsotg);
1491 		if (tr_type != DWC2_TRANSACTION_NONE)
1492 			dwc2_hcd_queue_transactions(hsotg, tr_type);
1493 	}
1494 
1495 	spin_unlock_irqrestore(&hsotg->lock, flags);
1496 	return HRTIMER_NORESTART;
1497 }
1498 
1499 /**
1500  * dwc2_qh_init() - Initializes a QH structure
1501  *
1502  * @hsotg: The HCD state structure for the DWC OTG controller
1503  * @qh:    The QH to init
1504  * @urb:   Holds the information about the device/endpoint needed to initialize
1505  *         the QH
1506  * @mem_flags: Flags for allocating memory.
1507  */
dwc2_qh_init(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,struct dwc2_hcd_urb * urb,gfp_t mem_flags)1508 static void dwc2_qh_init(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
1509 			 struct dwc2_hcd_urb *urb, gfp_t mem_flags)
1510 {
1511 	int dev_speed = dwc2_host_get_speed(hsotg, urb->priv);
1512 	u8 ep_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1513 	bool ep_is_in = !!dwc2_hcd_is_pipe_in(&urb->pipe_info);
1514 	bool ep_is_isoc = (ep_type == USB_ENDPOINT_XFER_ISOC);
1515 	bool ep_is_int = (ep_type == USB_ENDPOINT_XFER_INT);
1516 	u32 hprt = dwc2_readl(hsotg, HPRT0);
1517 	u32 prtspd = (hprt & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
1518 	bool do_split = (prtspd == HPRT0_SPD_HIGH_SPEED &&
1519 			 dev_speed != USB_SPEED_HIGH);
1520 	int maxp = dwc2_hcd_get_maxp(&urb->pipe_info);
1521 	int maxp_mult = dwc2_hcd_get_maxp_mult(&urb->pipe_info);
1522 	int bytecount = maxp_mult * maxp;
1523 	char *speed, *type;
1524 
1525 	/* Initialize QH */
1526 	qh->hsotg = hsotg;
1527 	timer_setup(&qh->unreserve_timer, dwc2_unreserve_timer_fn, 0);
1528 	hrtimer_init(&qh->wait_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1529 	qh->wait_timer.function = &dwc2_wait_timer_fn;
1530 	qh->ep_type = ep_type;
1531 	qh->ep_is_in = ep_is_in;
1532 
1533 	qh->data_toggle = DWC2_HC_PID_DATA0;
1534 	qh->maxp = maxp;
1535 	qh->maxp_mult = maxp_mult;
1536 	INIT_LIST_HEAD(&qh->qtd_list);
1537 	INIT_LIST_HEAD(&qh->qh_list_entry);
1538 
1539 	qh->do_split = do_split;
1540 	qh->dev_speed = dev_speed;
1541 
1542 	if (ep_is_int || ep_is_isoc) {
1543 		/* Compute scheduling parameters once and save them */
1544 		int host_speed = do_split ? USB_SPEED_HIGH : dev_speed;
1545 		struct dwc2_tt *dwc_tt = dwc2_host_get_tt_info(hsotg, urb->priv,
1546 							       mem_flags,
1547 							       &qh->ttport);
1548 		int device_ns;
1549 
1550 		qh->dwc_tt = dwc_tt;
1551 
1552 		qh->host_us = NS_TO_US(usb_calc_bus_time(host_speed, ep_is_in,
1553 				       ep_is_isoc, bytecount));
1554 		device_ns = usb_calc_bus_time(dev_speed, ep_is_in,
1555 					      ep_is_isoc, bytecount);
1556 
1557 		if (do_split && dwc_tt)
1558 			device_ns += dwc_tt->usb_tt->think_time;
1559 		qh->device_us = NS_TO_US(device_ns);
1560 
1561 		qh->device_interval = urb->interval;
1562 		qh->host_interval = urb->interval * (do_split ? 8 : 1);
1563 
1564 		/*
1565 		 * Schedule low speed if we're running the host in low or
1566 		 * full speed OR if we've got a "TT" to deal with to access this
1567 		 * device.
1568 		 */
1569 		qh->schedule_low_speed = prtspd != HPRT0_SPD_HIGH_SPEED ||
1570 					 dwc_tt;
1571 
1572 		if (do_split) {
1573 			/* We won't know num transfers until we schedule */
1574 			qh->num_hs_transfers = -1;
1575 		} else if (dev_speed == USB_SPEED_HIGH) {
1576 			qh->num_hs_transfers = 1;
1577 		} else {
1578 			qh->num_hs_transfers = 0;
1579 		}
1580 
1581 		/* We'll schedule later when we have something to do */
1582 	}
1583 
1584 	switch (dev_speed) {
1585 	case USB_SPEED_LOW:
1586 		speed = "low";
1587 		break;
1588 	case USB_SPEED_FULL:
1589 		speed = "full";
1590 		break;
1591 	case USB_SPEED_HIGH:
1592 		speed = "high";
1593 		break;
1594 	default:
1595 		speed = "?";
1596 		break;
1597 	}
1598 
1599 	switch (qh->ep_type) {
1600 	case USB_ENDPOINT_XFER_ISOC:
1601 		type = "isochronous";
1602 		break;
1603 	case USB_ENDPOINT_XFER_INT:
1604 		type = "interrupt";
1605 		break;
1606 	case USB_ENDPOINT_XFER_CONTROL:
1607 		type = "control";
1608 		break;
1609 	case USB_ENDPOINT_XFER_BULK:
1610 		type = "bulk";
1611 		break;
1612 	default:
1613 		type = "?";
1614 		break;
1615 	}
1616 
1617 	dwc2_sch_dbg(hsotg, "QH=%p Init %s, %s speed, %d bytes:\n", qh, type,
1618 		     speed, bytecount);
1619 	dwc2_sch_dbg(hsotg, "QH=%p ...addr=%d, ep=%d, %s\n", qh,
1620 		     dwc2_hcd_get_dev_addr(&urb->pipe_info),
1621 		     dwc2_hcd_get_ep_num(&urb->pipe_info),
1622 		     ep_is_in ? "IN" : "OUT");
1623 	if (ep_is_int || ep_is_isoc) {
1624 		dwc2_sch_dbg(hsotg,
1625 			     "QH=%p ...duration: host=%d us, device=%d us\n",
1626 			     qh, qh->host_us, qh->device_us);
1627 		dwc2_sch_dbg(hsotg, "QH=%p ...interval: host=%d, device=%d\n",
1628 			     qh, qh->host_interval, qh->device_interval);
1629 		if (qh->schedule_low_speed)
1630 			dwc2_sch_dbg(hsotg, "QH=%p ...low speed schedule=%p\n",
1631 				     qh, dwc2_get_ls_map(hsotg, qh));
1632 	}
1633 }
1634 
1635 /**
1636  * dwc2_hcd_qh_create() - Allocates and initializes a QH
1637  *
1638  * @hsotg:        The HCD state structure for the DWC OTG controller
1639  * @urb:          Holds the information about the device/endpoint needed
1640  *                to initialize the QH
1641  * @mem_flags:   Flags for allocating memory.
1642  *
1643  * Return: Pointer to the newly allocated QH, or NULL on error
1644  */
dwc2_hcd_qh_create(struct dwc2_hsotg * hsotg,struct dwc2_hcd_urb * urb,gfp_t mem_flags)1645 struct dwc2_qh *dwc2_hcd_qh_create(struct dwc2_hsotg *hsotg,
1646 				   struct dwc2_hcd_urb *urb,
1647 					  gfp_t mem_flags)
1648 {
1649 	struct dwc2_qh *qh;
1650 
1651 	if (!urb->priv)
1652 		return NULL;
1653 
1654 	/* Allocate memory */
1655 	qh = kzalloc(sizeof(*qh), mem_flags);
1656 	if (!qh)
1657 		return NULL;
1658 
1659 	dwc2_qh_init(hsotg, qh, urb, mem_flags);
1660 
1661 	if (hsotg->params.dma_desc_enable &&
1662 	    dwc2_hcd_qh_init_ddma(hsotg, qh, mem_flags) < 0) {
1663 		dwc2_hcd_qh_free(hsotg, qh);
1664 		return NULL;
1665 	}
1666 
1667 	return qh;
1668 }
1669 
1670 /**
1671  * dwc2_hcd_qh_free() - Frees the QH
1672  *
1673  * @hsotg: HCD instance
1674  * @qh:    The QH to free
1675  *
1676  * QH should already be removed from the list. QTD list should already be empty
1677  * if called from URB Dequeue.
1678  *
1679  * Must NOT be called with interrupt disabled or spinlock held
1680  */
dwc2_hcd_qh_free(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1681 void dwc2_hcd_qh_free(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1682 {
1683 	/* Make sure any unreserve work is finished. */
1684 	if (del_timer_sync(&qh->unreserve_timer)) {
1685 		unsigned long flags;
1686 
1687 		spin_lock_irqsave(&hsotg->lock, flags);
1688 		dwc2_do_unreserve(hsotg, qh);
1689 		spin_unlock_irqrestore(&hsotg->lock, flags);
1690 	}
1691 
1692 	/*
1693 	 * We don't have the lock so we can safely wait until the wait timer
1694 	 * finishes.  Of course, at this point in time we'd better have set
1695 	 * wait_timer_active to false so if this timer was still pending it
1696 	 * won't do anything anyway, but we want it to finish before we free
1697 	 * memory.
1698 	 */
1699 	hrtimer_cancel(&qh->wait_timer);
1700 
1701 	dwc2_host_put_tt_info(hsotg, qh->dwc_tt);
1702 
1703 	if (qh->desc_list)
1704 		dwc2_hcd_qh_free_ddma(hsotg, qh);
1705 	else if (hsotg->unaligned_cache && qh->dw_align_buf)
1706 		kmem_cache_free(hsotg->unaligned_cache, qh->dw_align_buf);
1707 
1708 	kfree(qh);
1709 }
1710 
1711 /**
1712  * dwc2_hcd_qh_add() - Adds a QH to either the non periodic or periodic
1713  * schedule if it is not already in the schedule. If the QH is already in
1714  * the schedule, no action is taken.
1715  *
1716  * @hsotg: The HCD state structure for the DWC OTG controller
1717  * @qh:    The QH to add
1718  *
1719  * Return: 0 if successful, negative error code otherwise
1720  */
dwc2_hcd_qh_add(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1721 int dwc2_hcd_qh_add(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1722 {
1723 	int status;
1724 	u32 intr_mask;
1725 	ktime_t delay;
1726 
1727 	if (dbg_qh(qh))
1728 		dev_vdbg(hsotg->dev, "%s()\n", __func__);
1729 
1730 	if (!list_empty(&qh->qh_list_entry))
1731 		/* QH already in a schedule */
1732 		return 0;
1733 
1734 	/* Add the new QH to the appropriate schedule */
1735 	if (dwc2_qh_is_non_per(qh)) {
1736 		/* Schedule right away */
1737 		qh->start_active_frame = hsotg->frame_number;
1738 		qh->next_active_frame = qh->start_active_frame;
1739 
1740 		if (qh->want_wait) {
1741 			list_add_tail(&qh->qh_list_entry,
1742 				      &hsotg->non_periodic_sched_waiting);
1743 			qh->wait_timer_cancel = false;
1744 			delay = ktime_set(0, DWC2_RETRY_WAIT_DELAY);
1745 			hrtimer_start(&qh->wait_timer, delay, HRTIMER_MODE_REL);
1746 		} else {
1747 			list_add_tail(&qh->qh_list_entry,
1748 				      &hsotg->non_periodic_sched_inactive);
1749 		}
1750 		return 0;
1751 	}
1752 
1753 	status = dwc2_schedule_periodic(hsotg, qh);
1754 	if (status)
1755 		return status;
1756 	if (!hsotg->periodic_qh_count) {
1757 		intr_mask = dwc2_readl(hsotg, GINTMSK);
1758 		intr_mask |= GINTSTS_SOF;
1759 		dwc2_writel(hsotg, intr_mask, GINTMSK);
1760 	}
1761 	hsotg->periodic_qh_count++;
1762 
1763 	return 0;
1764 }
1765 
1766 /**
1767  * dwc2_hcd_qh_unlink() - Removes a QH from either the non-periodic or periodic
1768  * schedule. Memory is not freed.
1769  *
1770  * @hsotg: The HCD state structure
1771  * @qh:    QH to remove from schedule
1772  */
dwc2_hcd_qh_unlink(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)1773 void dwc2_hcd_qh_unlink(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
1774 {
1775 	u32 intr_mask;
1776 
1777 	dev_vdbg(hsotg->dev, "%s()\n", __func__);
1778 
1779 	/* If the wait_timer is pending, this will stop it from acting */
1780 	qh->wait_timer_cancel = true;
1781 
1782 	if (list_empty(&qh->qh_list_entry))
1783 		/* QH is not in a schedule */
1784 		return;
1785 
1786 	if (dwc2_qh_is_non_per(qh)) {
1787 		if (hsotg->non_periodic_qh_ptr == &qh->qh_list_entry)
1788 			hsotg->non_periodic_qh_ptr =
1789 					hsotg->non_periodic_qh_ptr->next;
1790 		list_del_init(&qh->qh_list_entry);
1791 		return;
1792 	}
1793 
1794 	dwc2_deschedule_periodic(hsotg, qh);
1795 	hsotg->periodic_qh_count--;
1796 	if (!hsotg->periodic_qh_count &&
1797 	    !hsotg->params.dma_desc_enable) {
1798 		intr_mask = dwc2_readl(hsotg, GINTMSK);
1799 		intr_mask &= ~GINTSTS_SOF;
1800 		dwc2_writel(hsotg, intr_mask, GINTMSK);
1801 	}
1802 }
1803 
1804 /**
1805  * dwc2_next_for_periodic_split() - Set next_active_frame midway thru a split.
1806  *
1807  * This is called for setting next_active_frame for periodic splits for all but
1808  * the first packet of the split.  Confusing?  I thought so...
1809  *
1810  * Periodic splits are single low/full speed transfers that we end up splitting
1811  * up into several high speed transfers.  They always fit into one full (1 ms)
1812  * frame but might be split over several microframes (125 us each).  We to put
1813  * each of the parts on a very specific high speed frame.
1814  *
1815  * This function figures out where the next active uFrame needs to be.
1816  *
1817  * @hsotg:        The HCD state structure
1818  * @qh:           QH for the periodic transfer.
1819  * @frame_number: The current frame number.
1820  *
1821  * Return: number missed by (or 0 if we didn't miss).
1822  */
dwc2_next_for_periodic_split(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,u16 frame_number)1823 static int dwc2_next_for_periodic_split(struct dwc2_hsotg *hsotg,
1824 					struct dwc2_qh *qh, u16 frame_number)
1825 {
1826 	u16 old_frame = qh->next_active_frame;
1827 	u16 prev_frame_number = dwc2_frame_num_dec(frame_number, 1);
1828 	int missed = 0;
1829 	u16 incr;
1830 
1831 	/*
1832 	 * See dwc2_uframe_schedule_split() for split scheduling.
1833 	 *
1834 	 * Basically: increment 1 normally, but 2 right after the start split
1835 	 * (except for ISOC out).
1836 	 */
1837 	if (old_frame == qh->start_active_frame &&
1838 	    !(qh->ep_type == USB_ENDPOINT_XFER_ISOC && !qh->ep_is_in))
1839 		incr = 2;
1840 	else
1841 		incr = 1;
1842 
1843 	qh->next_active_frame = dwc2_frame_num_inc(old_frame, incr);
1844 
1845 	/*
1846 	 * Note that it's OK for frame_number to be 1 frame past
1847 	 * next_active_frame.  Remember that next_active_frame is supposed to
1848 	 * be 1 frame _before_ when we want to be scheduled.  If we're 1 frame
1849 	 * past it just means schedule ASAP.
1850 	 *
1851 	 * It's _not_ OK, however, if we're more than one frame past.
1852 	 */
1853 	if (dwc2_frame_num_gt(prev_frame_number, qh->next_active_frame)) {
1854 		/*
1855 		 * OOPS, we missed.  That's actually pretty bad since
1856 		 * the hub will be unhappy; try ASAP I guess.
1857 		 */
1858 		missed = dwc2_frame_num_dec(prev_frame_number,
1859 					    qh->next_active_frame);
1860 		qh->next_active_frame = frame_number;
1861 	}
1862 
1863 	return missed;
1864 }
1865 
1866 /**
1867  * dwc2_next_periodic_start() - Set next_active_frame for next transfer start
1868  *
1869  * This is called for setting next_active_frame for a periodic transfer for
1870  * all cases other than midway through a periodic split.  This will also update
1871  * start_active_frame.
1872  *
1873  * Since we _always_ keep start_active_frame as the start of the previous
1874  * transfer this is normally pretty easy: we just add our interval to
1875  * start_active_frame and we've got our answer.
1876  *
1877  * The tricks come into play if we miss.  In that case we'll look for the next
1878  * slot we can fit into.
1879  *
1880  * @hsotg:        The HCD state structure
1881  * @qh:           QH for the periodic transfer.
1882  * @frame_number: The current frame number.
1883  *
1884  * Return: number missed by (or 0 if we didn't miss).
1885  */
dwc2_next_periodic_start(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,u16 frame_number)1886 static int dwc2_next_periodic_start(struct dwc2_hsotg *hsotg,
1887 				    struct dwc2_qh *qh, u16 frame_number)
1888 {
1889 	int missed = 0;
1890 	u16 interval = qh->host_interval;
1891 	u16 prev_frame_number = dwc2_frame_num_dec(frame_number, 1);
1892 
1893 	qh->start_active_frame = dwc2_frame_num_inc(qh->start_active_frame,
1894 						    interval);
1895 
1896 	/*
1897 	 * The dwc2_frame_num_gt() function used below won't work terribly well
1898 	 * with if we just incremented by a really large intervals since the
1899 	 * frame counter only goes to 0x3fff.  It's terribly unlikely that we
1900 	 * will have missed in this case anyway.  Just go to exit.  If we want
1901 	 * to try to do better we'll need to keep track of a bigger counter
1902 	 * somewhere in the driver and handle overflows.
1903 	 */
1904 	if (interval >= 0x1000)
1905 		goto exit;
1906 
1907 	/*
1908 	 * Test for misses, which is when it's too late to schedule.
1909 	 *
1910 	 * A few things to note:
1911 	 * - We compare against prev_frame_number since start_active_frame
1912 	 *   and next_active_frame are always 1 frame before we want things
1913 	 *   to be active and we assume we can still get scheduled in the
1914 	 *   current frame number.
1915 	 * - It's possible for start_active_frame (now incremented) to be
1916 	 *   next_active_frame if we got an EO MISS (even_odd miss) which
1917 	 *   basically means that we detected there wasn't enough time for
1918 	 *   the last packet and dwc2_hc_set_even_odd_frame() rescheduled us
1919 	 *   at the last second.  We want to make sure we don't schedule
1920 	 *   another transfer for the same frame.  My test webcam doesn't seem
1921 	 *   terribly upset by missing a transfer but really doesn't like when
1922 	 *   we do two transfers in the same frame.
1923 	 * - Some misses are expected.  Specifically, in order to work
1924 	 *   perfectly dwc2 really needs quite spectacular interrupt latency
1925 	 *   requirements.  It needs to be able to handle its interrupts
1926 	 *   completely within 125 us of them being asserted. That not only
1927 	 *   means that the dwc2 interrupt handler needs to be fast but it
1928 	 *   means that nothing else in the system has to block dwc2 for a long
1929 	 *   time.  We can help with the dwc2 parts of this, but it's hard to
1930 	 *   guarantee that a system will have interrupt latency < 125 us, so
1931 	 *   we have to be robust to some misses.
1932 	 */
1933 	if (qh->start_active_frame == qh->next_active_frame ||
1934 	    dwc2_frame_num_gt(prev_frame_number, qh->start_active_frame)) {
1935 		u16 ideal_start = qh->start_active_frame;
1936 		int periods_in_map;
1937 
1938 		/*
1939 		 * Adjust interval as per gcd with map size.
1940 		 * See pmap_schedule() for more details here.
1941 		 */
1942 		if (qh->do_split || qh->dev_speed == USB_SPEED_HIGH)
1943 			periods_in_map = DWC2_HS_SCHEDULE_UFRAMES;
1944 		else
1945 			periods_in_map = DWC2_LS_SCHEDULE_FRAMES;
1946 		interval = gcd(interval, periods_in_map);
1947 
1948 		do {
1949 			qh->start_active_frame = dwc2_frame_num_inc(
1950 				qh->start_active_frame, interval);
1951 		} while (dwc2_frame_num_gt(prev_frame_number,
1952 					   qh->start_active_frame));
1953 
1954 		missed = dwc2_frame_num_dec(qh->start_active_frame,
1955 					    ideal_start);
1956 	}
1957 
1958 exit:
1959 	qh->next_active_frame = qh->start_active_frame;
1960 
1961 	return missed;
1962 }
1963 
1964 /*
1965  * Deactivates a QH. For non-periodic QHs, removes the QH from the active
1966  * non-periodic schedule. The QH is added to the inactive non-periodic
1967  * schedule if any QTDs are still attached to the QH.
1968  *
1969  * For periodic QHs, the QH is removed from the periodic queued schedule. If
1970  * there are any QTDs still attached to the QH, the QH is added to either the
1971  * periodic inactive schedule or the periodic ready schedule and its next
1972  * scheduled frame is calculated. The QH is placed in the ready schedule if
1973  * the scheduled frame has been reached already. Otherwise it's placed in the
1974  * inactive schedule. If there are no QTDs attached to the QH, the QH is
1975  * completely removed from the periodic schedule.
1976  */
dwc2_hcd_qh_deactivate(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,int sched_next_periodic_split)1977 void dwc2_hcd_qh_deactivate(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
1978 			    int sched_next_periodic_split)
1979 {
1980 	u16 old_frame = qh->next_active_frame;
1981 	u16 frame_number;
1982 	int missed;
1983 
1984 	if (dbg_qh(qh))
1985 		dev_vdbg(hsotg->dev, "%s()\n", __func__);
1986 
1987 	if (dwc2_qh_is_non_per(qh)) {
1988 		dwc2_hcd_qh_unlink(hsotg, qh);
1989 		if (!list_empty(&qh->qtd_list))
1990 			/* Add back to inactive/waiting non-periodic schedule */
1991 			dwc2_hcd_qh_add(hsotg, qh);
1992 		return;
1993 	}
1994 
1995 	/*
1996 	 * Use the real frame number rather than the cached value as of the
1997 	 * last SOF just to get us a little closer to reality.  Note that
1998 	 * means we don't actually know if we've already handled the SOF
1999 	 * interrupt for this frame.
2000 	 */
2001 	frame_number = dwc2_hcd_get_frame_number(hsotg);
2002 
2003 	if (sched_next_periodic_split)
2004 		missed = dwc2_next_for_periodic_split(hsotg, qh, frame_number);
2005 	else
2006 		missed = dwc2_next_periodic_start(hsotg, qh, frame_number);
2007 
2008 	dwc2_sch_vdbg(hsotg,
2009 		      "QH=%p next(%d) fn=%04x, sch=%04x=>%04x (%+d) miss=%d %s\n",
2010 		     qh, sched_next_periodic_split, frame_number, old_frame,
2011 		     qh->next_active_frame,
2012 		     dwc2_frame_num_dec(qh->next_active_frame, old_frame),
2013 		missed, missed ? "MISS" : "");
2014 
2015 	if (list_empty(&qh->qtd_list)) {
2016 		dwc2_hcd_qh_unlink(hsotg, qh);
2017 		return;
2018 	}
2019 
2020 	/*
2021 	 * Remove from periodic_sched_queued and move to
2022 	 * appropriate queue
2023 	 *
2024 	 * Note: we purposely use the frame_number from the "hsotg" structure
2025 	 * since we know SOF interrupt will handle future frames.
2026 	 */
2027 	if (dwc2_frame_num_le(qh->next_active_frame, hsotg->frame_number))
2028 		list_move_tail(&qh->qh_list_entry,
2029 			       &hsotg->periodic_sched_ready);
2030 	else
2031 		list_move_tail(&qh->qh_list_entry,
2032 			       &hsotg->periodic_sched_inactive);
2033 }
2034 
2035 /**
2036  * dwc2_hcd_qtd_init() - Initializes a QTD structure
2037  *
2038  * @qtd: The QTD to initialize
2039  * @urb: The associated URB
2040  */
dwc2_hcd_qtd_init(struct dwc2_qtd * qtd,struct dwc2_hcd_urb * urb)2041 void dwc2_hcd_qtd_init(struct dwc2_qtd *qtd, struct dwc2_hcd_urb *urb)
2042 {
2043 	qtd->urb = urb;
2044 	if (dwc2_hcd_get_pipe_type(&urb->pipe_info) ==
2045 			USB_ENDPOINT_XFER_CONTROL) {
2046 		/*
2047 		 * The only time the QTD data toggle is used is on the data
2048 		 * phase of control transfers. This phase always starts with
2049 		 * DATA1.
2050 		 */
2051 		qtd->data_toggle = DWC2_HC_PID_DATA1;
2052 		qtd->control_phase = DWC2_CONTROL_SETUP;
2053 	}
2054 
2055 	/* Start split */
2056 	qtd->complete_split = 0;
2057 	qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
2058 	qtd->isoc_split_offset = 0;
2059 	qtd->in_process = 0;
2060 
2061 	/* Store the qtd ptr in the urb to reference the QTD */
2062 	urb->qtd = qtd;
2063 }
2064 
2065 /**
2066  * dwc2_hcd_qtd_add() - Adds a QTD to the QTD-list of a QH
2067  *			Caller must hold driver lock.
2068  *
2069  * @hsotg:        The DWC HCD structure
2070  * @qtd:          The QTD to add
2071  * @qh:           Queue head to add qtd to
2072  *
2073  * Return: 0 if successful, negative error code otherwise
2074  *
2075  * If the QH to which the QTD is added is not currently scheduled, it is placed
2076  * into the proper schedule based on its EP type.
2077  */
dwc2_hcd_qtd_add(struct dwc2_hsotg * hsotg,struct dwc2_qtd * qtd,struct dwc2_qh * qh)2078 int dwc2_hcd_qtd_add(struct dwc2_hsotg *hsotg, struct dwc2_qtd *qtd,
2079 		     struct dwc2_qh *qh)
2080 {
2081 	int retval;
2082 
2083 	if (unlikely(!qh)) {
2084 		dev_err(hsotg->dev, "%s: Invalid QH\n", __func__);
2085 		retval = -EINVAL;
2086 		goto fail;
2087 	}
2088 
2089 	retval = dwc2_hcd_qh_add(hsotg, qh);
2090 	if (retval)
2091 		goto fail;
2092 
2093 	qtd->qh = qh;
2094 	list_add_tail(&qtd->qtd_list_entry, &qh->qtd_list);
2095 
2096 	return 0;
2097 fail:
2098 	return retval;
2099 }
2100