1 /*
2  * xHCI host controller driver
3  *
4  * Copyright (C) 2008 Intel Corp.
5  *
6  * Author: Sarah Sharp
7  * Some code borrowed from the Linux EHCI driver.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
16  * for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21  */
22 
23 #include <linux/pci.h>
24 #include <linux/irq.h>
25 #include <linux/log2.h>
26 #include <linux/module.h>
27 #include <linux/moduleparam.h>
28 #include <linux/slab.h>
29 
30 #include "xhci.h"
31 
32 #define DRIVER_AUTHOR "Sarah Sharp"
33 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
34 
35 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
36 static int link_quirk;
37 module_param(link_quirk, int, S_IRUGO | S_IWUSR);
38 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
39 
40 /* TODO: copied from ehci-hcd.c - can this be refactored? */
41 /*
42  * handshake - spin reading hc until handshake completes or fails
43  * @ptr: address of hc register to be read
44  * @mask: bits to look at in result of read
45  * @done: value of those bits when handshake succeeds
46  * @usec: timeout in microseconds
47  *
48  * Returns negative errno, or zero on success
49  *
50  * Success happens when the "mask" bits have the specified value (hardware
51  * handshake done).  There are two failure modes:  "usec" have passed (major
52  * hardware flakeout), or the register reads as all-ones (hardware removed).
53  */
handshake(struct xhci_hcd * xhci,void __iomem * ptr,u32 mask,u32 done,int usec)54 static int handshake(struct xhci_hcd *xhci, void __iomem *ptr,
55 		      u32 mask, u32 done, int usec)
56 {
57 	u32	result;
58 
59 	do {
60 		result = xhci_readl(xhci, ptr);
61 		if (result == ~(u32)0)		/* card removed */
62 			return -ENODEV;
63 		result &= mask;
64 		if (result == done)
65 			return 0;
66 		udelay(1);
67 		usec--;
68 	} while (usec > 0);
69 	return -ETIMEDOUT;
70 }
71 
72 /*
73  * Disable interrupts and begin the xHCI halting process.
74  */
xhci_quiesce(struct xhci_hcd * xhci)75 void xhci_quiesce(struct xhci_hcd *xhci)
76 {
77 	u32 halted;
78 	u32 cmd;
79 	u32 mask;
80 
81 	mask = ~(XHCI_IRQS);
82 	halted = xhci_readl(xhci, &xhci->op_regs->status) & STS_HALT;
83 	if (!halted)
84 		mask &= ~CMD_RUN;
85 
86 	cmd = xhci_readl(xhci, &xhci->op_regs->command);
87 	cmd &= mask;
88 	xhci_writel(xhci, cmd, &xhci->op_regs->command);
89 }
90 
91 /*
92  * Force HC into halt state.
93  *
94  * Disable any IRQs and clear the run/stop bit.
95  * HC will complete any current and actively pipelined transactions, and
96  * should halt within 16 ms of the run/stop bit being cleared.
97  * Read HC Halted bit in the status register to see when the HC is finished.
98  */
xhci_halt(struct xhci_hcd * xhci)99 int xhci_halt(struct xhci_hcd *xhci)
100 {
101 	int ret;
102 	xhci_dbg(xhci, "// Halt the HC\n");
103 	xhci_quiesce(xhci);
104 
105 	ret = handshake(xhci, &xhci->op_regs->status,
106 			STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
107 	if (!ret)
108 		xhci->xhc_state |= XHCI_STATE_HALTED;
109 	return ret;
110 }
111 
112 /*
113  * Set the run bit and wait for the host to be running.
114  */
xhci_start(struct xhci_hcd * xhci)115 static int xhci_start(struct xhci_hcd *xhci)
116 {
117 	u32 temp;
118 	int ret;
119 
120 	temp = xhci_readl(xhci, &xhci->op_regs->command);
121 	temp |= (CMD_RUN);
122 	xhci_dbg(xhci, "// Turn on HC, cmd = 0x%x.\n",
123 			temp);
124 	xhci_writel(xhci, temp, &xhci->op_regs->command);
125 
126 	/*
127 	 * Wait for the HCHalted Status bit to be 0 to indicate the host is
128 	 * running.
129 	 */
130 	ret = handshake(xhci, &xhci->op_regs->status,
131 			STS_HALT, 0, XHCI_MAX_HALT_USEC);
132 	if (ret == -ETIMEDOUT)
133 		xhci_err(xhci, "Host took too long to start, "
134 				"waited %u microseconds.\n",
135 				XHCI_MAX_HALT_USEC);
136 	if (!ret)
137 		xhci->xhc_state &= ~XHCI_STATE_HALTED;
138 	return ret;
139 }
140 
141 /*
142  * Reset a halted HC.
143  *
144  * This resets pipelines, timers, counters, state machines, etc.
145  * Transactions will be terminated immediately, and operational registers
146  * will be set to their defaults.
147  */
xhci_reset(struct xhci_hcd * xhci)148 int xhci_reset(struct xhci_hcd *xhci)
149 {
150 	u32 command;
151 	u32 state;
152 	int ret;
153 
154 	state = xhci_readl(xhci, &xhci->op_regs->status);
155 	if ((state & STS_HALT) == 0) {
156 		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
157 		return 0;
158 	}
159 
160 	xhci_dbg(xhci, "// Reset the HC\n");
161 	command = xhci_readl(xhci, &xhci->op_regs->command);
162 	command |= CMD_RESET;
163 	xhci_writel(xhci, command, &xhci->op_regs->command);
164 
165 	ret = handshake(xhci, &xhci->op_regs->command,
166 			CMD_RESET, 0, 250 * 1000);
167 	if (ret)
168 		return ret;
169 
170 	xhci_dbg(xhci, "Wait for controller to be ready for doorbell rings\n");
171 	/*
172 	 * xHCI cannot write to any doorbells or operational registers other
173 	 * than status until the "Controller Not Ready" flag is cleared.
174 	 */
175 	return handshake(xhci, &xhci->op_regs->status, STS_CNR, 0, 250 * 1000);
176 }
177 
178 /*
179  * Free IRQs
180  * free all IRQs request
181  */
xhci_free_irq(struct xhci_hcd * xhci)182 static void xhci_free_irq(struct xhci_hcd *xhci)
183 {
184 	int i;
185 	struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
186 
187 	/* return if using legacy interrupt */
188 	if (xhci_to_hcd(xhci)->irq >= 0)
189 		return;
190 
191 	if (xhci->msix_entries) {
192 		for (i = 0; i < xhci->msix_count; i++)
193 			if (xhci->msix_entries[i].vector)
194 				free_irq(xhci->msix_entries[i].vector,
195 						xhci_to_hcd(xhci));
196 	} else if (pdev->irq >= 0)
197 		free_irq(pdev->irq, xhci_to_hcd(xhci));
198 
199 	return;
200 }
201 
202 /*
203  * Set up MSI
204  */
xhci_setup_msi(struct xhci_hcd * xhci)205 static int xhci_setup_msi(struct xhci_hcd *xhci)
206 {
207 	int ret;
208 	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
209 
210 	ret = pci_enable_msi(pdev);
211 	if (ret) {
212 		xhci_err(xhci, "failed to allocate MSI entry\n");
213 		return ret;
214 	}
215 
216 	ret = request_irq(pdev->irq, (irq_handler_t)xhci_msi_irq,
217 				0, "xhci_hcd", xhci_to_hcd(xhci));
218 	if (ret) {
219 		xhci_err(xhci, "disable MSI interrupt\n");
220 		pci_disable_msi(pdev);
221 	}
222 
223 	return ret;
224 }
225 
226 /*
227  * Set up MSI-X
228  */
xhci_setup_msix(struct xhci_hcd * xhci)229 static int xhci_setup_msix(struct xhci_hcd *xhci)
230 {
231 	int i, ret = 0;
232 	struct usb_hcd *hcd = xhci_to_hcd(xhci);
233 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
234 
235 	/*
236 	 * calculate number of msi-x vectors supported.
237 	 * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
238 	 *   with max number of interrupters based on the xhci HCSPARAMS1.
239 	 * - num_online_cpus: maximum msi-x vectors per CPUs core.
240 	 *   Add additional 1 vector to ensure always available interrupt.
241 	 */
242 	xhci->msix_count = min(num_online_cpus() + 1,
243 				HCS_MAX_INTRS(xhci->hcs_params1));
244 
245 	xhci->msix_entries =
246 		kmalloc((sizeof(struct msix_entry))*xhci->msix_count,
247 				GFP_KERNEL);
248 	if (!xhci->msix_entries) {
249 		xhci_err(xhci, "Failed to allocate MSI-X entries\n");
250 		return -ENOMEM;
251 	}
252 
253 	for (i = 0; i < xhci->msix_count; i++) {
254 		xhci->msix_entries[i].entry = i;
255 		xhci->msix_entries[i].vector = 0;
256 	}
257 
258 	ret = pci_enable_msix(pdev, xhci->msix_entries, xhci->msix_count);
259 	if (ret) {
260 		xhci_err(xhci, "Failed to enable MSI-X\n");
261 		goto free_entries;
262 	}
263 
264 	for (i = 0; i < xhci->msix_count; i++) {
265 		ret = request_irq(xhci->msix_entries[i].vector,
266 				(irq_handler_t)xhci_msi_irq,
267 				0, "xhci_hcd", xhci_to_hcd(xhci));
268 		if (ret)
269 			goto disable_msix;
270 	}
271 
272 	hcd->msix_enabled = 1;
273 	return ret;
274 
275 disable_msix:
276 	xhci_err(xhci, "disable MSI-X interrupt\n");
277 	xhci_free_irq(xhci);
278 	pci_disable_msix(pdev);
279 free_entries:
280 	kfree(xhci->msix_entries);
281 	xhci->msix_entries = NULL;
282 	return ret;
283 }
284 
285 /* Free any IRQs and disable MSI-X */
xhci_cleanup_msix(struct xhci_hcd * xhci)286 static void xhci_cleanup_msix(struct xhci_hcd *xhci)
287 {
288 	struct usb_hcd *hcd = xhci_to_hcd(xhci);
289 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
290 
291 	xhci_free_irq(xhci);
292 
293 	if (xhci->msix_entries) {
294 		pci_disable_msix(pdev);
295 		kfree(xhci->msix_entries);
296 		xhci->msix_entries = NULL;
297 	} else {
298 		pci_disable_msi(pdev);
299 	}
300 
301 	hcd->msix_enabled = 0;
302 	return;
303 }
304 
305 /*
306  * Initialize memory for HCD and xHC (one-time init).
307  *
308  * Program the PAGESIZE register, initialize the device context array, create
309  * device contexts (?), set up a command ring segment (or two?), create event
310  * ring (one for now).
311  */
xhci_init(struct usb_hcd * hcd)312 int xhci_init(struct usb_hcd *hcd)
313 {
314 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
315 	int retval = 0;
316 
317 	xhci_dbg(xhci, "xhci_init\n");
318 	spin_lock_init(&xhci->lock);
319 	if (link_quirk) {
320 		xhci_dbg(xhci, "QUIRK: Not clearing Link TRB chain bits.\n");
321 		xhci->quirks |= XHCI_LINK_TRB_QUIRK;
322 	} else {
323 		xhci_dbg(xhci, "xHCI doesn't need link TRB QUIRK\n");
324 	}
325 	retval = xhci_mem_init(xhci, GFP_KERNEL);
326 	xhci_dbg(xhci, "Finished xhci_init\n");
327 
328 	return retval;
329 }
330 
331 /*-------------------------------------------------------------------------*/
332 
333 
334 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
xhci_event_ring_work(unsigned long arg)335 static void xhci_event_ring_work(unsigned long arg)
336 {
337 	unsigned long flags;
338 	int temp;
339 	u64 temp_64;
340 	struct xhci_hcd *xhci = (struct xhci_hcd *) arg;
341 	int i, j;
342 
343 	xhci_dbg(xhci, "Poll event ring: %lu\n", jiffies);
344 
345 	spin_lock_irqsave(&xhci->lock, flags);
346 	temp = xhci_readl(xhci, &xhci->op_regs->status);
347 	xhci_dbg(xhci, "op reg status = 0x%x\n", temp);
348 	if (temp == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING)) {
349 		xhci_dbg(xhci, "HW died, polling stopped.\n");
350 		spin_unlock_irqrestore(&xhci->lock, flags);
351 		return;
352 	}
353 
354 	temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
355 	xhci_dbg(xhci, "ir_set 0 pending = 0x%x\n", temp);
356 	xhci_dbg(xhci, "HC error bitmask = 0x%x\n", xhci->error_bitmask);
357 	xhci->error_bitmask = 0;
358 	xhci_dbg(xhci, "Event ring:\n");
359 	xhci_debug_segment(xhci, xhci->event_ring->deq_seg);
360 	xhci_dbg_ring_ptrs(xhci, xhci->event_ring);
361 	temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
362 	temp_64 &= ~ERST_PTR_MASK;
363 	xhci_dbg(xhci, "ERST deq = 64'h%0lx\n", (long unsigned int) temp_64);
364 	xhci_dbg(xhci, "Command ring:\n");
365 	xhci_debug_segment(xhci, xhci->cmd_ring->deq_seg);
366 	xhci_dbg_ring_ptrs(xhci, xhci->cmd_ring);
367 	xhci_dbg_cmd_ptrs(xhci);
368 	for (i = 0; i < MAX_HC_SLOTS; ++i) {
369 		if (!xhci->devs[i])
370 			continue;
371 		for (j = 0; j < 31; ++j) {
372 			xhci_dbg_ep_rings(xhci, i, j, &xhci->devs[i]->eps[j]);
373 		}
374 	}
375 	spin_unlock_irqrestore(&xhci->lock, flags);
376 
377 	if (!xhci->zombie)
378 		mod_timer(&xhci->event_ring_timer, jiffies + POLL_TIMEOUT * HZ);
379 	else
380 		xhci_dbg(xhci, "Quit polling the event ring.\n");
381 }
382 #endif
383 
xhci_run_finished(struct xhci_hcd * xhci)384 static int xhci_run_finished(struct xhci_hcd *xhci)
385 {
386 	if (xhci_start(xhci)) {
387 		xhci_halt(xhci);
388 		return -ENODEV;
389 	}
390 	xhci->shared_hcd->state = HC_STATE_RUNNING;
391 
392 	if (xhci->quirks & XHCI_NEC_HOST)
393 		xhci_ring_cmd_db(xhci);
394 
395 	xhci_dbg(xhci, "Finished xhci_run for USB3 roothub\n");
396 	return 0;
397 }
398 
399 /*
400  * Start the HC after it was halted.
401  *
402  * This function is called by the USB core when the HC driver is added.
403  * Its opposite is xhci_stop().
404  *
405  * xhci_init() must be called once before this function can be called.
406  * Reset the HC, enable device slot contexts, program DCBAAP, and
407  * set command ring pointer and event ring pointer.
408  *
409  * Setup MSI-X vectors and enable interrupts.
410  */
xhci_run(struct usb_hcd * hcd)411 int xhci_run(struct usb_hcd *hcd)
412 {
413 	u32 temp;
414 	u64 temp_64;
415 	u32 ret;
416 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
417 	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
418 
419 	/* Start the xHCI host controller running only after the USB 2.0 roothub
420 	 * is setup.
421 	 */
422 
423 	hcd->uses_new_polling = 1;
424 	if (!usb_hcd_is_primary_hcd(hcd))
425 		return xhci_run_finished(xhci);
426 
427 	xhci_dbg(xhci, "xhci_run\n");
428 	/* unregister the legacy interrupt */
429 	if (hcd->irq)
430 		free_irq(hcd->irq, hcd);
431 	hcd->irq = -1;
432 
433 	ret = xhci_setup_msix(xhci);
434 	if (ret)
435 		/* fall back to msi*/
436 		ret = xhci_setup_msi(xhci);
437 
438 	if (ret) {
439 		/* fall back to legacy interrupt*/
440 		ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
441 					hcd->irq_descr, hcd);
442 		if (ret) {
443 			xhci_err(xhci, "request interrupt %d failed\n",
444 					pdev->irq);
445 			return ret;
446 		}
447 		hcd->irq = pdev->irq;
448 	}
449 
450 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
451 	init_timer(&xhci->event_ring_timer);
452 	xhci->event_ring_timer.data = (unsigned long) xhci;
453 	xhci->event_ring_timer.function = xhci_event_ring_work;
454 	/* Poll the event ring */
455 	xhci->event_ring_timer.expires = jiffies + POLL_TIMEOUT * HZ;
456 	xhci->zombie = 0;
457 	xhci_dbg(xhci, "Setting event ring polling timer\n");
458 	add_timer(&xhci->event_ring_timer);
459 #endif
460 
461 	xhci_dbg(xhci, "Command ring memory map follows:\n");
462 	xhci_debug_ring(xhci, xhci->cmd_ring);
463 	xhci_dbg_ring_ptrs(xhci, xhci->cmd_ring);
464 	xhci_dbg_cmd_ptrs(xhci);
465 
466 	xhci_dbg(xhci, "ERST memory map follows:\n");
467 	xhci_dbg_erst(xhci, &xhci->erst);
468 	xhci_dbg(xhci, "Event ring:\n");
469 	xhci_debug_ring(xhci, xhci->event_ring);
470 	xhci_dbg_ring_ptrs(xhci, xhci->event_ring);
471 	temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
472 	temp_64 &= ~ERST_PTR_MASK;
473 	xhci_dbg(xhci, "ERST deq = 64'h%0lx\n", (long unsigned int) temp_64);
474 
475 	xhci_dbg(xhci, "// Set the interrupt modulation register\n");
476 	temp = xhci_readl(xhci, &xhci->ir_set->irq_control);
477 	temp &= ~ER_IRQ_INTERVAL_MASK;
478 	temp |= (u32) 160;
479 	xhci_writel(xhci, temp, &xhci->ir_set->irq_control);
480 
481 	/* Set the HCD state before we enable the irqs */
482 	temp = xhci_readl(xhci, &xhci->op_regs->command);
483 	temp |= (CMD_EIE);
484 	xhci_dbg(xhci, "// Enable interrupts, cmd = 0x%x.\n",
485 			temp);
486 	xhci_writel(xhci, temp, &xhci->op_regs->command);
487 
488 	temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
489 	xhci_dbg(xhci, "// Enabling event ring interrupter %p by writing 0x%x to irq_pending\n",
490 			xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
491 	xhci_writel(xhci, ER_IRQ_ENABLE(temp),
492 			&xhci->ir_set->irq_pending);
493 	xhci_print_ir_set(xhci, 0);
494 
495 	if (xhci->quirks & XHCI_NEC_HOST)
496 		xhci_queue_vendor_command(xhci, 0, 0, 0,
497 				TRB_TYPE(TRB_NEC_GET_FW));
498 
499 	xhci_dbg(xhci, "Finished xhci_run for USB2 roothub\n");
500 	return 0;
501 }
502 
xhci_only_stop_hcd(struct usb_hcd * hcd)503 static void xhci_only_stop_hcd(struct usb_hcd *hcd)
504 {
505 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
506 
507 	spin_lock_irq(&xhci->lock);
508 	xhci_halt(xhci);
509 
510 	/* The shared_hcd is going to be deallocated shortly (the USB core only
511 	 * calls this function when allocation fails in usb_add_hcd(), or
512 	 * usb_remove_hcd() is called).  So we need to unset xHCI's pointer.
513 	 */
514 	xhci->shared_hcd = NULL;
515 	spin_unlock_irq(&xhci->lock);
516 }
517 
518 /*
519  * Stop xHCI driver.
520  *
521  * This function is called by the USB core when the HC driver is removed.
522  * Its opposite is xhci_run().
523  *
524  * Disable device contexts, disable IRQs, and quiesce the HC.
525  * Reset the HC, finish any completed transactions, and cleanup memory.
526  */
xhci_stop(struct usb_hcd * hcd)527 void xhci_stop(struct usb_hcd *hcd)
528 {
529 	u32 temp;
530 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
531 
532 	if (!usb_hcd_is_primary_hcd(hcd)) {
533 		xhci_only_stop_hcd(xhci->shared_hcd);
534 		return;
535 	}
536 
537 	spin_lock_irq(&xhci->lock);
538 	/* Make sure the xHC is halted for a USB3 roothub
539 	 * (xhci_stop() could be called as part of failed init).
540 	 */
541 	xhci_halt(xhci);
542 	xhci_reset(xhci);
543 	spin_unlock_irq(&xhci->lock);
544 
545 	xhci_cleanup_msix(xhci);
546 
547 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
548 	/* Tell the event ring poll function not to reschedule */
549 	xhci->zombie = 1;
550 	del_timer_sync(&xhci->event_ring_timer);
551 #endif
552 
553 	if (xhci->quirks & XHCI_AMD_PLL_FIX)
554 		usb_amd_dev_put();
555 
556 	xhci_dbg(xhci, "// Disabling event ring interrupts\n");
557 	temp = xhci_readl(xhci, &xhci->op_regs->status);
558 	xhci_writel(xhci, temp & ~STS_EINT, &xhci->op_regs->status);
559 	temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
560 	xhci_writel(xhci, ER_IRQ_DISABLE(temp),
561 			&xhci->ir_set->irq_pending);
562 	xhci_print_ir_set(xhci, 0);
563 
564 	xhci_dbg(xhci, "cleaning up memory\n");
565 	xhci_mem_cleanup(xhci);
566 	xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
567 		    xhci_readl(xhci, &xhci->op_regs->status));
568 }
569 
570 /*
571  * Shutdown HC (not bus-specific)
572  *
573  * This is called when the machine is rebooting or halting.  We assume that the
574  * machine will be powered off, and the HC's internal state will be reset.
575  * Don't bother to free memory.
576  *
577  * This will only ever be called with the main usb_hcd (the USB3 roothub).
578  */
xhci_shutdown(struct usb_hcd * hcd)579 void xhci_shutdown(struct usb_hcd *hcd)
580 {
581 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
582 
583 	spin_lock_irq(&xhci->lock);
584 	xhci_halt(xhci);
585 	spin_unlock_irq(&xhci->lock);
586 
587 	xhci_cleanup_msix(xhci);
588 
589 	xhci_dbg(xhci, "xhci_shutdown completed - status = %x\n",
590 		    xhci_readl(xhci, &xhci->op_regs->status));
591 }
592 
593 #ifdef CONFIG_PM
xhci_save_registers(struct xhci_hcd * xhci)594 static void xhci_save_registers(struct xhci_hcd *xhci)
595 {
596 	xhci->s3.command = xhci_readl(xhci, &xhci->op_regs->command);
597 	xhci->s3.dev_nt = xhci_readl(xhci, &xhci->op_regs->dev_notification);
598 	xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
599 	xhci->s3.config_reg = xhci_readl(xhci, &xhci->op_regs->config_reg);
600 	xhci->s3.irq_pending = xhci_readl(xhci, &xhci->ir_set->irq_pending);
601 	xhci->s3.irq_control = xhci_readl(xhci, &xhci->ir_set->irq_control);
602 	xhci->s3.erst_size = xhci_readl(xhci, &xhci->ir_set->erst_size);
603 	xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
604 	xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
605 }
606 
xhci_restore_registers(struct xhci_hcd * xhci)607 static void xhci_restore_registers(struct xhci_hcd *xhci)
608 {
609 	xhci_writel(xhci, xhci->s3.command, &xhci->op_regs->command);
610 	xhci_writel(xhci, xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
611 	xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
612 	xhci_writel(xhci, xhci->s3.config_reg, &xhci->op_regs->config_reg);
613 	xhci_writel(xhci, xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
614 	xhci_writel(xhci, xhci->s3.irq_control, &xhci->ir_set->irq_control);
615 	xhci_writel(xhci, xhci->s3.erst_size, &xhci->ir_set->erst_size);
616 	xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
617 }
618 
xhci_set_cmd_ring_deq(struct xhci_hcd * xhci)619 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
620 {
621 	u64	val_64;
622 
623 	/* step 2: initialize command ring buffer */
624 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
625 	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
626 		(xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
627 				      xhci->cmd_ring->dequeue) &
628 		 (u64) ~CMD_RING_RSVD_BITS) |
629 		xhci->cmd_ring->cycle_state;
630 	xhci_dbg(xhci, "// Setting command ring address to 0x%llx\n",
631 			(long unsigned long) val_64);
632 	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
633 }
634 
635 /*
636  * The whole command ring must be cleared to zero when we suspend the host.
637  *
638  * The host doesn't save the command ring pointer in the suspend well, so we
639  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
640  * aligned, because of the reserved bits in the command ring dequeue pointer
641  * register.  Therefore, we can't just set the dequeue pointer back in the
642  * middle of the ring (TRBs are 16-byte aligned).
643  */
xhci_clear_command_ring(struct xhci_hcd * xhci)644 static void xhci_clear_command_ring(struct xhci_hcd *xhci)
645 {
646 	struct xhci_ring *ring;
647 	struct xhci_segment *seg;
648 
649 	ring = xhci->cmd_ring;
650 	seg = ring->deq_seg;
651 	do {
652 		memset(seg->trbs, 0, SEGMENT_SIZE);
653 		seg = seg->next;
654 	} while (seg != ring->deq_seg);
655 
656 	/* Reset the software enqueue and dequeue pointers */
657 	ring->deq_seg = ring->first_seg;
658 	ring->dequeue = ring->first_seg->trbs;
659 	ring->enq_seg = ring->deq_seg;
660 	ring->enqueue = ring->dequeue;
661 
662 	/*
663 	 * Ring is now zeroed, so the HW should look for change of ownership
664 	 * when the cycle bit is set to 1.
665 	 */
666 	ring->cycle_state = 1;
667 
668 	/*
669 	 * Reset the hardware dequeue pointer.
670 	 * Yes, this will need to be re-written after resume, but we're paranoid
671 	 * and want to make sure the hardware doesn't access bogus memory
672 	 * because, say, the BIOS or an SMI started the host without changing
673 	 * the command ring pointers.
674 	 */
675 	xhci_set_cmd_ring_deq(xhci);
676 }
677 
678 /*
679  * Stop HC (not bus-specific)
680  *
681  * This is called when the machine transition into S3/S4 mode.
682  *
683  */
xhci_suspend(struct xhci_hcd * xhci)684 int xhci_suspend(struct xhci_hcd *xhci)
685 {
686 	int			rc = 0;
687 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
688 	u32			command;
689 	int			i;
690 
691 	spin_lock_irq(&xhci->lock);
692 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
693 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
694 	/* step 1: stop endpoint */
695 	/* skipped assuming that port suspend has done */
696 
697 	/* step 2: clear Run/Stop bit */
698 	command = xhci_readl(xhci, &xhci->op_regs->command);
699 	command &= ~CMD_RUN;
700 	xhci_writel(xhci, command, &xhci->op_regs->command);
701 	if (handshake(xhci, &xhci->op_regs->status,
702 		      STS_HALT, STS_HALT, 100*100)) {
703 		xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
704 		spin_unlock_irq(&xhci->lock);
705 		return -ETIMEDOUT;
706 	}
707 	xhci_clear_command_ring(xhci);
708 
709 	/* step 3: save registers */
710 	xhci_save_registers(xhci);
711 
712 	/* step 4: set CSS flag */
713 	command = xhci_readl(xhci, &xhci->op_regs->command);
714 	command |= CMD_CSS;
715 	xhci_writel(xhci, command, &xhci->op_regs->command);
716 	if (handshake(xhci, &xhci->op_regs->status, STS_SAVE, 0, 10*100)) {
717 		xhci_warn(xhci, "WARN: xHC CMD_CSS timeout\n");
718 		spin_unlock_irq(&xhci->lock);
719 		return -ETIMEDOUT;
720 	}
721 	spin_unlock_irq(&xhci->lock);
722 
723 	/* step 5: remove core well power */
724 	/* synchronize irq when using MSI-X */
725 	if (xhci->msix_entries) {
726 		for (i = 0; i < xhci->msix_count; i++)
727 			synchronize_irq(xhci->msix_entries[i].vector);
728 	}
729 
730 	return rc;
731 }
732 
733 /*
734  * start xHC (not bus-specific)
735  *
736  * This is called when the machine transition from S3/S4 mode.
737  *
738  */
xhci_resume(struct xhci_hcd * xhci,bool hibernated)739 int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
740 {
741 	u32			command, temp = 0;
742 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
743 	struct usb_hcd		*secondary_hcd;
744 	int			retval;
745 
746 	/* Wait a bit if either of the roothubs need to settle from the
747 	 * transition into bus suspend.
748 	 */
749 	if (time_before(jiffies, xhci->bus_state[0].next_statechange) ||
750 			time_before(jiffies,
751 				xhci->bus_state[1].next_statechange))
752 		msleep(100);
753 
754 	spin_lock_irq(&xhci->lock);
755 
756 	if (!hibernated) {
757 		/* step 1: restore register */
758 		xhci_restore_registers(xhci);
759 		/* step 2: initialize command ring buffer */
760 		xhci_set_cmd_ring_deq(xhci);
761 		/* step 3: restore state and start state*/
762 		/* step 3: set CRS flag */
763 		command = xhci_readl(xhci, &xhci->op_regs->command);
764 		command |= CMD_CRS;
765 		xhci_writel(xhci, command, &xhci->op_regs->command);
766 		if (handshake(xhci, &xhci->op_regs->status,
767 			      STS_RESTORE, 0, 10*100)) {
768 			xhci_dbg(xhci, "WARN: xHC CMD_CSS timeout\n");
769 			spin_unlock_irq(&xhci->lock);
770 			return -ETIMEDOUT;
771 		}
772 		temp = xhci_readl(xhci, &xhci->op_regs->status);
773 	}
774 
775 	/* If restore operation fails, re-initialize the HC during resume */
776 	if ((temp & STS_SRE) || hibernated) {
777 		/* Let the USB core know _both_ roothubs lost power. */
778 		usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
779 		usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
780 
781 		xhci_dbg(xhci, "Stop HCD\n");
782 		xhci_halt(xhci);
783 		xhci_reset(xhci);
784 		spin_unlock_irq(&xhci->lock);
785 		xhci_cleanup_msix(xhci);
786 
787 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
788 		/* Tell the event ring poll function not to reschedule */
789 		xhci->zombie = 1;
790 		del_timer_sync(&xhci->event_ring_timer);
791 #endif
792 
793 		xhci_dbg(xhci, "// Disabling event ring interrupts\n");
794 		temp = xhci_readl(xhci, &xhci->op_regs->status);
795 		xhci_writel(xhci, temp & ~STS_EINT, &xhci->op_regs->status);
796 		temp = xhci_readl(xhci, &xhci->ir_set->irq_pending);
797 		xhci_writel(xhci, ER_IRQ_DISABLE(temp),
798 				&xhci->ir_set->irq_pending);
799 		xhci_print_ir_set(xhci, 0);
800 
801 		xhci_dbg(xhci, "cleaning up memory\n");
802 		xhci_mem_cleanup(xhci);
803 		xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
804 			    xhci_readl(xhci, &xhci->op_regs->status));
805 
806 		/* USB core calls the PCI reinit and start functions twice:
807 		 * first with the primary HCD, and then with the secondary HCD.
808 		 * If we don't do the same, the host will never be started.
809 		 */
810 		if (!usb_hcd_is_primary_hcd(hcd))
811 			secondary_hcd = hcd;
812 		else
813 			secondary_hcd = xhci->shared_hcd;
814 
815 		xhci_dbg(xhci, "Initialize the xhci_hcd\n");
816 		retval = xhci_init(hcd->primary_hcd);
817 		if (retval)
818 			return retval;
819 		xhci_dbg(xhci, "Start the primary HCD\n");
820 		retval = xhci_run(hcd->primary_hcd);
821 		if (retval)
822 			goto failed_restart;
823 
824 		xhci_dbg(xhci, "Start the secondary HCD\n");
825 		retval = xhci_run(secondary_hcd);
826 		if (!retval) {
827 			set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
828 			set_bit(HCD_FLAG_HW_ACCESSIBLE,
829 					&xhci->shared_hcd->flags);
830 		}
831 failed_restart:
832 		hcd->state = HC_STATE_SUSPENDED;
833 		xhci->shared_hcd->state = HC_STATE_SUSPENDED;
834 		return retval;
835 	}
836 
837 	/* step 4: set Run/Stop bit */
838 	command = xhci_readl(xhci, &xhci->op_regs->command);
839 	command |= CMD_RUN;
840 	xhci_writel(xhci, command, &xhci->op_regs->command);
841 	handshake(xhci, &xhci->op_regs->status, STS_HALT,
842 		  0, 250 * 1000);
843 
844 	/* step 5: walk topology and initialize portsc,
845 	 * portpmsc and portli
846 	 */
847 	/* this is done in bus_resume */
848 
849 	/* step 6: restart each of the previously
850 	 * Running endpoints by ringing their doorbells
851 	 */
852 
853 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
854 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
855 
856 	spin_unlock_irq(&xhci->lock);
857 	return 0;
858 }
859 #endif	/* CONFIG_PM */
860 
861 /*-------------------------------------------------------------------------*/
862 
863 /**
864  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
865  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
866  * value to right shift 1 for the bitmask.
867  *
868  * Index  = (epnum * 2) + direction - 1,
869  * where direction = 0 for OUT, 1 for IN.
870  * For control endpoints, the IN index is used (OUT index is unused), so
871  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
872  */
xhci_get_endpoint_index(struct usb_endpoint_descriptor * desc)873 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
874 {
875 	unsigned int index;
876 	if (usb_endpoint_xfer_control(desc))
877 		index = (unsigned int) (usb_endpoint_num(desc)*2);
878 	else
879 		index = (unsigned int) (usb_endpoint_num(desc)*2) +
880 			(usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
881 	return index;
882 }
883 
884 /* Find the flag for this endpoint (for use in the control context).  Use the
885  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
886  * bit 1, etc.
887  */
xhci_get_endpoint_flag(struct usb_endpoint_descriptor * desc)888 unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
889 {
890 	return 1 << (xhci_get_endpoint_index(desc) + 1);
891 }
892 
893 /* Find the flag for this endpoint (for use in the control context).  Use the
894  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
895  * bit 1, etc.
896  */
xhci_get_endpoint_flag_from_index(unsigned int ep_index)897 unsigned int xhci_get_endpoint_flag_from_index(unsigned int ep_index)
898 {
899 	return 1 << (ep_index + 1);
900 }
901 
902 /* Compute the last valid endpoint context index.  Basically, this is the
903  * endpoint index plus one.  For slot contexts with more than valid endpoint,
904  * we find the most significant bit set in the added contexts flags.
905  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
906  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
907  */
xhci_last_valid_endpoint(u32 added_ctxs)908 unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
909 {
910 	return fls(added_ctxs) - 1;
911 }
912 
913 /* Returns 1 if the arguments are OK;
914  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
915  */
xhci_check_args(struct usb_hcd * hcd,struct usb_device * udev,struct usb_host_endpoint * ep,int check_ep,bool check_virt_dev,const char * func)916 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
917 		struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
918 		const char *func) {
919 	struct xhci_hcd	*xhci;
920 	struct xhci_virt_device	*virt_dev;
921 
922 	if (!hcd || (check_ep && !ep) || !udev) {
923 		printk(KERN_DEBUG "xHCI %s called with invalid args\n",
924 				func);
925 		return -EINVAL;
926 	}
927 	if (!udev->parent) {
928 		printk(KERN_DEBUG "xHCI %s called for root hub\n",
929 				func);
930 		return 0;
931 	}
932 
933 	if (check_virt_dev) {
934 		xhci = hcd_to_xhci(hcd);
935 		if (!udev->slot_id || !xhci->devs
936 			|| !xhci->devs[udev->slot_id]) {
937 			printk(KERN_DEBUG "xHCI %s called with unaddressed "
938 						"device\n", func);
939 			return -EINVAL;
940 		}
941 
942 		virt_dev = xhci->devs[udev->slot_id];
943 		if (virt_dev->udev != udev) {
944 			printk(KERN_DEBUG "xHCI %s called with udev and "
945 					  "virt_dev does not match\n", func);
946 			return -EINVAL;
947 		}
948 	}
949 
950 	return 1;
951 }
952 
953 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
954 		struct usb_device *udev, struct xhci_command *command,
955 		bool ctx_change, bool must_succeed);
956 
957 /*
958  * Full speed devices may have a max packet size greater than 8 bytes, but the
959  * USB core doesn't know that until it reads the first 8 bytes of the
960  * descriptor.  If the usb_device's max packet size changes after that point,
961  * we need to issue an evaluate context command and wait on it.
962  */
xhci_check_maxpacket(struct xhci_hcd * xhci,unsigned int slot_id,unsigned int ep_index,struct urb * urb)963 static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
964 		unsigned int ep_index, struct urb *urb)
965 {
966 	struct xhci_container_ctx *in_ctx;
967 	struct xhci_container_ctx *out_ctx;
968 	struct xhci_input_control_ctx *ctrl_ctx;
969 	struct xhci_ep_ctx *ep_ctx;
970 	int max_packet_size;
971 	int hw_max_packet_size;
972 	int ret = 0;
973 
974 	out_ctx = xhci->devs[slot_id]->out_ctx;
975 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
976 	hw_max_packet_size = MAX_PACKET_DECODED(ep_ctx->ep_info2);
977 	max_packet_size = urb->dev->ep0.desc.wMaxPacketSize;
978 	if (hw_max_packet_size != max_packet_size) {
979 		xhci_dbg(xhci, "Max Packet Size for ep 0 changed.\n");
980 		xhci_dbg(xhci, "Max packet size in usb_device = %d\n",
981 				max_packet_size);
982 		xhci_dbg(xhci, "Max packet size in xHCI HW = %d\n",
983 				hw_max_packet_size);
984 		xhci_dbg(xhci, "Issuing evaluate context command.\n");
985 
986 		/* Set up the modified control endpoint 0 */
987 		xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
988 				xhci->devs[slot_id]->out_ctx, ep_index);
989 		in_ctx = xhci->devs[slot_id]->in_ctx;
990 		ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
991 		ep_ctx->ep_info2 &= ~MAX_PACKET_MASK;
992 		ep_ctx->ep_info2 |= MAX_PACKET(max_packet_size);
993 
994 		/* Set up the input context flags for the command */
995 		/* FIXME: This won't work if a non-default control endpoint
996 		 * changes max packet sizes.
997 		 */
998 		ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
999 		ctrl_ctx->add_flags = EP0_FLAG;
1000 		ctrl_ctx->drop_flags = 0;
1001 
1002 		xhci_dbg(xhci, "Slot %d input context\n", slot_id);
1003 		xhci_dbg_ctx(xhci, in_ctx, ep_index);
1004 		xhci_dbg(xhci, "Slot %d output context\n", slot_id);
1005 		xhci_dbg_ctx(xhci, out_ctx, ep_index);
1006 
1007 		ret = xhci_configure_endpoint(xhci, urb->dev, NULL,
1008 				true, false);
1009 
1010 		/* Clean up the input context for later use by bandwidth
1011 		 * functions.
1012 		 */
1013 		ctrl_ctx->add_flags = SLOT_FLAG;
1014 	}
1015 	return ret;
1016 }
1017 
1018 /*
1019  * non-error returns are a promise to giveback() the urb later
1020  * we drop ownership so next owner (or urb unlink) can get it
1021  */
xhci_urb_enqueue(struct usb_hcd * hcd,struct urb * urb,gfp_t mem_flags)1022 int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1023 {
1024 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1025 	unsigned long flags;
1026 	int ret = 0;
1027 	unsigned int slot_id, ep_index;
1028 	struct urb_priv	*urb_priv;
1029 	int size, i;
1030 
1031 	if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
1032 					true, true, __func__) <= 0)
1033 		return -EINVAL;
1034 
1035 	slot_id = urb->dev->slot_id;
1036 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1037 
1038 	if (!HCD_HW_ACCESSIBLE(hcd)) {
1039 		if (!in_interrupt())
1040 			xhci_dbg(xhci, "urb submitted during PCI suspend\n");
1041 		ret = -ESHUTDOWN;
1042 		goto exit;
1043 	}
1044 
1045 	if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1046 		size = urb->number_of_packets;
1047 	else
1048 		size = 1;
1049 
1050 	urb_priv = kzalloc(sizeof(struct urb_priv) +
1051 				  size * sizeof(struct xhci_td *), mem_flags);
1052 	if (!urb_priv)
1053 		return -ENOMEM;
1054 
1055 	for (i = 0; i < size; i++) {
1056 		urb_priv->td[i] = kzalloc(sizeof(struct xhci_td), mem_flags);
1057 		if (!urb_priv->td[i]) {
1058 			urb_priv->length = i;
1059 			xhci_urb_free_priv(xhci, urb_priv);
1060 			return -ENOMEM;
1061 		}
1062 	}
1063 
1064 	urb_priv->length = size;
1065 	urb_priv->td_cnt = 0;
1066 	urb->hcpriv = urb_priv;
1067 
1068 	if (usb_endpoint_xfer_control(&urb->ep->desc)) {
1069 		/* Check to see if the max packet size for the default control
1070 		 * endpoint changed during FS device enumeration
1071 		 */
1072 		if (urb->dev->speed == USB_SPEED_FULL) {
1073 			ret = xhci_check_maxpacket(xhci, slot_id,
1074 					ep_index, urb);
1075 			if (ret < 0)
1076 				return ret;
1077 		}
1078 
1079 		/* We have a spinlock and interrupts disabled, so we must pass
1080 		 * atomic context to this function, which may allocate memory.
1081 		 */
1082 		spin_lock_irqsave(&xhci->lock, flags);
1083 		if (xhci->xhc_state & XHCI_STATE_DYING)
1084 			goto dying;
1085 		ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1086 				slot_id, ep_index);
1087 		spin_unlock_irqrestore(&xhci->lock, flags);
1088 	} else if (usb_endpoint_xfer_bulk(&urb->ep->desc)) {
1089 		spin_lock_irqsave(&xhci->lock, flags);
1090 		if (xhci->xhc_state & XHCI_STATE_DYING)
1091 			goto dying;
1092 		if (xhci->devs[slot_id]->eps[ep_index].ep_state &
1093 				EP_GETTING_STREAMS) {
1094 			xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
1095 					"is transitioning to using streams.\n");
1096 			ret = -EINVAL;
1097 		} else if (xhci->devs[slot_id]->eps[ep_index].ep_state &
1098 				EP_GETTING_NO_STREAMS) {
1099 			xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
1100 					"is transitioning to "
1101 					"not having streams.\n");
1102 			ret = -EINVAL;
1103 		} else {
1104 			ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1105 					slot_id, ep_index);
1106 		}
1107 		spin_unlock_irqrestore(&xhci->lock, flags);
1108 	} else if (usb_endpoint_xfer_int(&urb->ep->desc)) {
1109 		spin_lock_irqsave(&xhci->lock, flags);
1110 		if (xhci->xhc_state & XHCI_STATE_DYING)
1111 			goto dying;
1112 		ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1113 				slot_id, ep_index);
1114 		spin_unlock_irqrestore(&xhci->lock, flags);
1115 	} else {
1116 		spin_lock_irqsave(&xhci->lock, flags);
1117 		if (xhci->xhc_state & XHCI_STATE_DYING)
1118 			goto dying;
1119 		ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1120 				slot_id, ep_index);
1121 		spin_unlock_irqrestore(&xhci->lock, flags);
1122 	}
1123 exit:
1124 	return ret;
1125 dying:
1126 	xhci_urb_free_priv(xhci, urb_priv);
1127 	urb->hcpriv = NULL;
1128 	xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for "
1129 			"non-responsive xHCI host.\n",
1130 			urb->ep->desc.bEndpointAddress, urb);
1131 	spin_unlock_irqrestore(&xhci->lock, flags);
1132 	return -ESHUTDOWN;
1133 }
1134 
1135 /* Get the right ring for the given URB.
1136  * If the endpoint supports streams, boundary check the URB's stream ID.
1137  * If the endpoint doesn't support streams, return the singular endpoint ring.
1138  */
xhci_urb_to_transfer_ring(struct xhci_hcd * xhci,struct urb * urb)1139 static struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci,
1140 		struct urb *urb)
1141 {
1142 	unsigned int slot_id;
1143 	unsigned int ep_index;
1144 	unsigned int stream_id;
1145 	struct xhci_virt_ep *ep;
1146 
1147 	slot_id = urb->dev->slot_id;
1148 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1149 	stream_id = urb->stream_id;
1150 	ep = &xhci->devs[slot_id]->eps[ep_index];
1151 	/* Common case: no streams */
1152 	if (!(ep->ep_state & EP_HAS_STREAMS))
1153 		return ep->ring;
1154 
1155 	if (stream_id == 0) {
1156 		xhci_warn(xhci,
1157 				"WARN: Slot ID %u, ep index %u has streams, "
1158 				"but URB has no stream ID.\n",
1159 				slot_id, ep_index);
1160 		return NULL;
1161 	}
1162 
1163 	if (stream_id < ep->stream_info->num_streams)
1164 		return ep->stream_info->stream_rings[stream_id];
1165 
1166 	xhci_warn(xhci,
1167 			"WARN: Slot ID %u, ep index %u has "
1168 			"stream IDs 1 to %u allocated, "
1169 			"but stream ID %u is requested.\n",
1170 			slot_id, ep_index,
1171 			ep->stream_info->num_streams - 1,
1172 			stream_id);
1173 	return NULL;
1174 }
1175 
1176 /*
1177  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
1178  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
1179  * should pick up where it left off in the TD, unless a Set Transfer Ring
1180  * Dequeue Pointer is issued.
1181  *
1182  * The TRBs that make up the buffers for the canceled URB will be "removed" from
1183  * the ring.  Since the ring is a contiguous structure, they can't be physically
1184  * removed.  Instead, there are two options:
1185  *
1186  *  1) If the HC is in the middle of processing the URB to be canceled, we
1187  *     simply move the ring's dequeue pointer past those TRBs using the Set
1188  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
1189  *     when drivers timeout on the last submitted URB and attempt to cancel.
1190  *
1191  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
1192  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1193  *     HC will need to invalidate the any TRBs it has cached after the stop
1194  *     endpoint command, as noted in the xHCI 0.95 errata.
1195  *
1196  *  3) The TD may have completed by the time the Stop Endpoint Command
1197  *     completes, so software needs to handle that case too.
1198  *
1199  * This function should protect against the TD enqueueing code ringing the
1200  * doorbell while this code is waiting for a Stop Endpoint command to complete.
1201  * It also needs to account for multiple cancellations on happening at the same
1202  * time for the same endpoint.
1203  *
1204  * Note that this function can be called in any context, or so says
1205  * usb_hcd_unlink_urb()
1206  */
xhci_urb_dequeue(struct usb_hcd * hcd,struct urb * urb,int status)1207 int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1208 {
1209 	unsigned long flags;
1210 	int ret, i;
1211 	u32 temp;
1212 	struct xhci_hcd *xhci;
1213 	struct urb_priv	*urb_priv;
1214 	struct xhci_td *td;
1215 	unsigned int ep_index;
1216 	struct xhci_ring *ep_ring;
1217 	struct xhci_virt_ep *ep;
1218 
1219 	xhci = hcd_to_xhci(hcd);
1220 	spin_lock_irqsave(&xhci->lock, flags);
1221 	/* Make sure the URB hasn't completed or been unlinked already */
1222 	ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1223 	if (ret || !urb->hcpriv)
1224 		goto done;
1225 	temp = xhci_readl(xhci, &xhci->op_regs->status);
1226 	if (temp == 0xffffffff || (xhci->xhc_state & XHCI_STATE_HALTED)) {
1227 		xhci_dbg(xhci, "HW died, freeing TD.\n");
1228 		urb_priv = urb->hcpriv;
1229 
1230 		usb_hcd_unlink_urb_from_ep(hcd, urb);
1231 		spin_unlock_irqrestore(&xhci->lock, flags);
1232 		usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1233 		xhci_urb_free_priv(xhci, urb_priv);
1234 		return ret;
1235 	}
1236 	if (xhci->xhc_state & XHCI_STATE_DYING) {
1237 		xhci_dbg(xhci, "Ep 0x%x: URB %p to be canceled on "
1238 				"non-responsive xHCI host.\n",
1239 				urb->ep->desc.bEndpointAddress, urb);
1240 		/* Let the stop endpoint command watchdog timer (which set this
1241 		 * state) finish cleaning up the endpoint TD lists.  We must
1242 		 * have caught it in the middle of dropping a lock and giving
1243 		 * back an URB.
1244 		 */
1245 		goto done;
1246 	}
1247 
1248 	xhci_dbg(xhci, "Cancel URB %p\n", urb);
1249 	xhci_dbg(xhci, "Event ring:\n");
1250 	xhci_debug_ring(xhci, xhci->event_ring);
1251 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1252 	ep = &xhci->devs[urb->dev->slot_id]->eps[ep_index];
1253 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1254 	if (!ep_ring) {
1255 		ret = -EINVAL;
1256 		goto done;
1257 	}
1258 
1259 	xhci_dbg(xhci, "Endpoint ring:\n");
1260 	xhci_debug_ring(xhci, ep_ring);
1261 
1262 	urb_priv = urb->hcpriv;
1263 
1264 	for (i = urb_priv->td_cnt; i < urb_priv->length; i++) {
1265 		td = urb_priv->td[i];
1266 		list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
1267 	}
1268 
1269 	/* Queue a stop endpoint command, but only if this is
1270 	 * the first cancellation to be handled.
1271 	 */
1272 	if (!(ep->ep_state & EP_HALT_PENDING)) {
1273 		ep->ep_state |= EP_HALT_PENDING;
1274 		ep->stop_cmds_pending++;
1275 		ep->stop_cmd_timer.expires = jiffies +
1276 			XHCI_STOP_EP_CMD_TIMEOUT * HZ;
1277 		add_timer(&ep->stop_cmd_timer);
1278 		xhci_queue_stop_endpoint(xhci, urb->dev->slot_id, ep_index, 0);
1279 		xhci_ring_cmd_db(xhci);
1280 	}
1281 done:
1282 	spin_unlock_irqrestore(&xhci->lock, flags);
1283 	return ret;
1284 }
1285 
1286 /* Drop an endpoint from a new bandwidth configuration for this device.
1287  * Only one call to this function is allowed per endpoint before
1288  * check_bandwidth() or reset_bandwidth() must be called.
1289  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1290  * add the endpoint to the schedule with possibly new parameters denoted by a
1291  * different endpoint descriptor in usb_host_endpoint.
1292  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1293  * not allowed.
1294  *
1295  * The USB core will not allow URBs to be queued to an endpoint that is being
1296  * disabled, so there's no need for mutual exclusion to protect
1297  * the xhci->devs[slot_id] structure.
1298  */
xhci_drop_endpoint(struct usb_hcd * hcd,struct usb_device * udev,struct usb_host_endpoint * ep)1299 int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1300 		struct usb_host_endpoint *ep)
1301 {
1302 	struct xhci_hcd *xhci;
1303 	struct xhci_container_ctx *in_ctx, *out_ctx;
1304 	struct xhci_input_control_ctx *ctrl_ctx;
1305 	struct xhci_slot_ctx *slot_ctx;
1306 	unsigned int last_ctx;
1307 	unsigned int ep_index;
1308 	struct xhci_ep_ctx *ep_ctx;
1309 	u32 drop_flag;
1310 	u32 new_add_flags, new_drop_flags, new_slot_info;
1311 	int ret;
1312 
1313 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1314 	if (ret <= 0)
1315 		return ret;
1316 	xhci = hcd_to_xhci(hcd);
1317 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1318 
1319 	drop_flag = xhci_get_endpoint_flag(&ep->desc);
1320 	if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1321 		xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1322 				__func__, drop_flag);
1323 		return 0;
1324 	}
1325 
1326 	in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1327 	out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1328 	ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1329 	ep_index = xhci_get_endpoint_index(&ep->desc);
1330 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1331 	/* If the HC already knows the endpoint is disabled,
1332 	 * or the HCD has noted it is disabled, ignore this request
1333 	 */
1334 	if ((ep_ctx->ep_info & EP_STATE_MASK) == EP_STATE_DISABLED ||
1335 			ctrl_ctx->drop_flags & xhci_get_endpoint_flag(&ep->desc)) {
1336 		xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1337 				__func__, ep);
1338 		return 0;
1339 	}
1340 
1341 	ctrl_ctx->drop_flags |= drop_flag;
1342 	new_drop_flags = ctrl_ctx->drop_flags;
1343 
1344 	ctrl_ctx->add_flags &= ~drop_flag;
1345 	new_add_flags = ctrl_ctx->add_flags;
1346 
1347 	last_ctx = xhci_last_valid_endpoint(ctrl_ctx->add_flags);
1348 	slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
1349 	/* Update the last valid endpoint context, if we deleted the last one */
1350 	if ((slot_ctx->dev_info & LAST_CTX_MASK) > LAST_CTX(last_ctx)) {
1351 		slot_ctx->dev_info &= ~LAST_CTX_MASK;
1352 		slot_ctx->dev_info |= LAST_CTX(last_ctx);
1353 	}
1354 	new_slot_info = slot_ctx->dev_info;
1355 
1356 	xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1357 
1358 	xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x, new slot info = %#x\n",
1359 			(unsigned int) ep->desc.bEndpointAddress,
1360 			udev->slot_id,
1361 			(unsigned int) new_drop_flags,
1362 			(unsigned int) new_add_flags,
1363 			(unsigned int) new_slot_info);
1364 	return 0;
1365 }
1366 
1367 /* Add an endpoint to a new possible bandwidth configuration for this device.
1368  * Only one call to this function is allowed per endpoint before
1369  * check_bandwidth() or reset_bandwidth() must be called.
1370  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1371  * add the endpoint to the schedule with possibly new parameters denoted by a
1372  * different endpoint descriptor in usb_host_endpoint.
1373  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1374  * not allowed.
1375  *
1376  * The USB core will not allow URBs to be queued to an endpoint until the
1377  * configuration or alt setting is installed in the device, so there's no need
1378  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1379  */
xhci_add_endpoint(struct usb_hcd * hcd,struct usb_device * udev,struct usb_host_endpoint * ep)1380 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1381 		struct usb_host_endpoint *ep)
1382 {
1383 	struct xhci_hcd *xhci;
1384 	struct xhci_container_ctx *in_ctx, *out_ctx;
1385 	unsigned int ep_index;
1386 	struct xhci_ep_ctx *ep_ctx;
1387 	struct xhci_slot_ctx *slot_ctx;
1388 	struct xhci_input_control_ctx *ctrl_ctx;
1389 	u32 added_ctxs;
1390 	unsigned int last_ctx;
1391 	u32 new_add_flags, new_drop_flags, new_slot_info;
1392 	int ret = 0;
1393 
1394 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1395 	if (ret <= 0) {
1396 		/* So we won't queue a reset ep command for a root hub */
1397 		ep->hcpriv = NULL;
1398 		return ret;
1399 	}
1400 	xhci = hcd_to_xhci(hcd);
1401 
1402 	added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1403 	last_ctx = xhci_last_valid_endpoint(added_ctxs);
1404 	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1405 		/* FIXME when we have to issue an evaluate endpoint command to
1406 		 * deal with ep0 max packet size changing once we get the
1407 		 * descriptors
1408 		 */
1409 		xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1410 				__func__, added_ctxs);
1411 		return 0;
1412 	}
1413 
1414 	in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1415 	out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1416 	ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1417 	ep_index = xhci_get_endpoint_index(&ep->desc);
1418 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1419 	/* If the HCD has already noted the endpoint is enabled,
1420 	 * ignore this request.
1421 	 */
1422 	if (ctrl_ctx->add_flags & xhci_get_endpoint_flag(&ep->desc)) {
1423 		xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
1424 				__func__, ep);
1425 		return 0;
1426 	}
1427 
1428 	/*
1429 	 * Configuration and alternate setting changes must be done in
1430 	 * process context, not interrupt context (or so documenation
1431 	 * for usb_set_interface() and usb_set_configuration() claim).
1432 	 */
1433 	if (xhci_endpoint_init(xhci, xhci->devs[udev->slot_id],
1434 				udev, ep, GFP_NOIO) < 0) {
1435 		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
1436 				__func__, ep->desc.bEndpointAddress);
1437 		return -ENOMEM;
1438 	}
1439 
1440 	ctrl_ctx->add_flags |= added_ctxs;
1441 	new_add_flags = ctrl_ctx->add_flags;
1442 
1443 	/* If xhci_endpoint_disable() was called for this endpoint, but the
1444 	 * xHC hasn't been notified yet through the check_bandwidth() call,
1445 	 * this re-adds a new state for the endpoint from the new endpoint
1446 	 * descriptors.  We must drop and re-add this endpoint, so we leave the
1447 	 * drop flags alone.
1448 	 */
1449 	new_drop_flags = ctrl_ctx->drop_flags;
1450 
1451 	slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
1452 	/* Update the last valid endpoint context, if we just added one past */
1453 	if ((slot_ctx->dev_info & LAST_CTX_MASK) < LAST_CTX(last_ctx)) {
1454 		slot_ctx->dev_info &= ~LAST_CTX_MASK;
1455 		slot_ctx->dev_info |= LAST_CTX(last_ctx);
1456 	}
1457 	new_slot_info = slot_ctx->dev_info;
1458 
1459 	/* Store the usb_device pointer for later use */
1460 	ep->hcpriv = udev;
1461 
1462 	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x, new slot info = %#x\n",
1463 			(unsigned int) ep->desc.bEndpointAddress,
1464 			udev->slot_id,
1465 			(unsigned int) new_drop_flags,
1466 			(unsigned int) new_add_flags,
1467 			(unsigned int) new_slot_info);
1468 	return 0;
1469 }
1470 
xhci_zero_in_ctx(struct xhci_hcd * xhci,struct xhci_virt_device * virt_dev)1471 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
1472 {
1473 	struct xhci_input_control_ctx *ctrl_ctx;
1474 	struct xhci_ep_ctx *ep_ctx;
1475 	struct xhci_slot_ctx *slot_ctx;
1476 	int i;
1477 
1478 	/* When a device's add flag and drop flag are zero, any subsequent
1479 	 * configure endpoint command will leave that endpoint's state
1480 	 * untouched.  Make sure we don't leave any old state in the input
1481 	 * endpoint contexts.
1482 	 */
1483 	ctrl_ctx = xhci_get_input_control_ctx(xhci, virt_dev->in_ctx);
1484 	ctrl_ctx->drop_flags = 0;
1485 	ctrl_ctx->add_flags = 0;
1486 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
1487 	slot_ctx->dev_info &= ~LAST_CTX_MASK;
1488 	/* Endpoint 0 is always valid */
1489 	slot_ctx->dev_info |= LAST_CTX(1);
1490 	for (i = 1; i < 31; ++i) {
1491 		ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
1492 		ep_ctx->ep_info = 0;
1493 		ep_ctx->ep_info2 = 0;
1494 		ep_ctx->deq = 0;
1495 		ep_ctx->tx_info = 0;
1496 	}
1497 }
1498 
xhci_configure_endpoint_result(struct xhci_hcd * xhci,struct usb_device * udev,int * cmd_status)1499 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
1500 		struct usb_device *udev, int *cmd_status)
1501 {
1502 	int ret;
1503 
1504 	switch (*cmd_status) {
1505 	case COMP_ENOMEM:
1506 		dev_warn(&udev->dev, "Not enough host controller resources "
1507 				"for new device state.\n");
1508 		ret = -ENOMEM;
1509 		/* FIXME: can we allocate more resources for the HC? */
1510 		break;
1511 	case COMP_BW_ERR:
1512 		dev_warn(&udev->dev, "Not enough bandwidth "
1513 				"for new device state.\n");
1514 		ret = -ENOSPC;
1515 		/* FIXME: can we go back to the old state? */
1516 		break;
1517 	case COMP_TRB_ERR:
1518 		/* the HCD set up something wrong */
1519 		dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
1520 				"add flag = 1, "
1521 				"and endpoint is not disabled.\n");
1522 		ret = -EINVAL;
1523 		break;
1524 	case COMP_SUCCESS:
1525 		dev_dbg(&udev->dev, "Successful Endpoint Configure command\n");
1526 		ret = 0;
1527 		break;
1528 	default:
1529 		xhci_err(xhci, "ERROR: unexpected command completion "
1530 				"code 0x%x.\n", *cmd_status);
1531 		ret = -EINVAL;
1532 		break;
1533 	}
1534 	return ret;
1535 }
1536 
xhci_evaluate_context_result(struct xhci_hcd * xhci,struct usb_device * udev,int * cmd_status)1537 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
1538 		struct usb_device *udev, int *cmd_status)
1539 {
1540 	int ret;
1541 	struct xhci_virt_device *virt_dev = xhci->devs[udev->slot_id];
1542 
1543 	switch (*cmd_status) {
1544 	case COMP_EINVAL:
1545 		dev_warn(&udev->dev, "WARN: xHCI driver setup invalid evaluate "
1546 				"context command.\n");
1547 		ret = -EINVAL;
1548 		break;
1549 	case COMP_EBADSLT:
1550 		dev_warn(&udev->dev, "WARN: slot not enabled for"
1551 				"evaluate context command.\n");
1552 	case COMP_CTX_STATE:
1553 		dev_warn(&udev->dev, "WARN: invalid context state for "
1554 				"evaluate context command.\n");
1555 		xhci_dbg_ctx(xhci, virt_dev->out_ctx, 1);
1556 		ret = -EINVAL;
1557 		break;
1558 	case COMP_SUCCESS:
1559 		dev_dbg(&udev->dev, "Successful evaluate context command\n");
1560 		ret = 0;
1561 		break;
1562 	default:
1563 		xhci_err(xhci, "ERROR: unexpected command completion "
1564 				"code 0x%x.\n", *cmd_status);
1565 		ret = -EINVAL;
1566 		break;
1567 	}
1568 	return ret;
1569 }
1570 
1571 /* Issue a configure endpoint command or evaluate context command
1572  * and wait for it to finish.
1573  */
xhci_configure_endpoint(struct xhci_hcd * xhci,struct usb_device * udev,struct xhci_command * command,bool ctx_change,bool must_succeed)1574 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1575 		struct usb_device *udev,
1576 		struct xhci_command *command,
1577 		bool ctx_change, bool must_succeed)
1578 {
1579 	int ret;
1580 	int timeleft;
1581 	unsigned long flags;
1582 	struct xhci_container_ctx *in_ctx;
1583 	struct completion *cmd_completion;
1584 	int *cmd_status;
1585 	struct xhci_virt_device *virt_dev;
1586 
1587 	spin_lock_irqsave(&xhci->lock, flags);
1588 	virt_dev = xhci->devs[udev->slot_id];
1589 	if (command) {
1590 		in_ctx = command->in_ctx;
1591 		cmd_completion = command->completion;
1592 		cmd_status = &command->status;
1593 		command->command_trb = xhci->cmd_ring->enqueue;
1594 
1595 		/* Enqueue pointer can be left pointing to the link TRB,
1596 		 * we must handle that
1597 		 */
1598 		if ((command->command_trb->link.control & TRB_TYPE_BITMASK)
1599 				== TRB_TYPE(TRB_LINK))
1600 			command->command_trb =
1601 				xhci->cmd_ring->enq_seg->next->trbs;
1602 
1603 		list_add_tail(&command->cmd_list, &virt_dev->cmd_list);
1604 	} else {
1605 		in_ctx = virt_dev->in_ctx;
1606 		cmd_completion = &virt_dev->cmd_completion;
1607 		cmd_status = &virt_dev->cmd_status;
1608 	}
1609 	init_completion(cmd_completion);
1610 
1611 	if (!ctx_change)
1612 		ret = xhci_queue_configure_endpoint(xhci, in_ctx->dma,
1613 				udev->slot_id, must_succeed);
1614 	else
1615 		ret = xhci_queue_evaluate_context(xhci, in_ctx->dma,
1616 				udev->slot_id);
1617 	if (ret < 0) {
1618 		if (command)
1619 			list_del(&command->cmd_list);
1620 		spin_unlock_irqrestore(&xhci->lock, flags);
1621 		xhci_dbg(xhci, "FIXME allocate a new ring segment\n");
1622 		return -ENOMEM;
1623 	}
1624 	xhci_ring_cmd_db(xhci);
1625 	spin_unlock_irqrestore(&xhci->lock, flags);
1626 
1627 	/* Wait for the configure endpoint command to complete */
1628 	timeleft = wait_for_completion_interruptible_timeout(
1629 			cmd_completion,
1630 			USB_CTRL_SET_TIMEOUT);
1631 	if (timeleft <= 0) {
1632 		xhci_warn(xhci, "%s while waiting for %s command\n",
1633 				timeleft == 0 ? "Timeout" : "Signal",
1634 				ctx_change == 0 ?
1635 					"configure endpoint" :
1636 					"evaluate context");
1637 		/* FIXME cancel the configure endpoint command */
1638 		return -ETIME;
1639 	}
1640 
1641 	if (!ctx_change)
1642 		return xhci_configure_endpoint_result(xhci, udev, cmd_status);
1643 	return xhci_evaluate_context_result(xhci, udev, cmd_status);
1644 }
1645 
1646 /* Called after one or more calls to xhci_add_endpoint() or
1647  * xhci_drop_endpoint().  If this call fails, the USB core is expected
1648  * to call xhci_reset_bandwidth().
1649  *
1650  * Since we are in the middle of changing either configuration or
1651  * installing a new alt setting, the USB core won't allow URBs to be
1652  * enqueued for any endpoint on the old config or interface.  Nothing
1653  * else should be touching the xhci->devs[slot_id] structure, so we
1654  * don't need to take the xhci->lock for manipulating that.
1655  */
xhci_check_bandwidth(struct usb_hcd * hcd,struct usb_device * udev)1656 int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
1657 {
1658 	int i;
1659 	int ret = 0;
1660 	struct xhci_hcd *xhci;
1661 	struct xhci_virt_device	*virt_dev;
1662 	struct xhci_input_control_ctx *ctrl_ctx;
1663 	struct xhci_slot_ctx *slot_ctx;
1664 
1665 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
1666 	if (ret <= 0)
1667 		return ret;
1668 	xhci = hcd_to_xhci(hcd);
1669 
1670 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1671 	virt_dev = xhci->devs[udev->slot_id];
1672 
1673 	/* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
1674 	ctrl_ctx = xhci_get_input_control_ctx(xhci, virt_dev->in_ctx);
1675 	ctrl_ctx->add_flags |= SLOT_FLAG;
1676 	ctrl_ctx->add_flags &= ~EP0_FLAG;
1677 	ctrl_ctx->drop_flags &= ~SLOT_FLAG;
1678 	ctrl_ctx->drop_flags &= ~EP0_FLAG;
1679 	xhci_dbg(xhci, "New Input Control Context:\n");
1680 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
1681 	xhci_dbg_ctx(xhci, virt_dev->in_ctx,
1682 			LAST_CTX_TO_EP_NUM(slot_ctx->dev_info));
1683 
1684 	ret = xhci_configure_endpoint(xhci, udev, NULL,
1685 			false, false);
1686 	if (ret) {
1687 		/* Callee should call reset_bandwidth() */
1688 		return ret;
1689 	}
1690 
1691 	xhci_dbg(xhci, "Output context after successful config ep cmd:\n");
1692 	xhci_dbg_ctx(xhci, virt_dev->out_ctx,
1693 			LAST_CTX_TO_EP_NUM(slot_ctx->dev_info));
1694 
1695 	xhci_zero_in_ctx(xhci, virt_dev);
1696 	/* Install new rings and free or cache any old rings */
1697 	for (i = 1; i < 31; ++i) {
1698 		if (!virt_dev->eps[i].new_ring)
1699 			continue;
1700 		/* Only cache or free the old ring if it exists.
1701 		 * It may not if this is the first add of an endpoint.
1702 		 */
1703 		if (virt_dev->eps[i].ring) {
1704 			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
1705 		}
1706 		virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
1707 		virt_dev->eps[i].new_ring = NULL;
1708 	}
1709 
1710 	return ret;
1711 }
1712 
xhci_reset_bandwidth(struct usb_hcd * hcd,struct usb_device * udev)1713 void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
1714 {
1715 	struct xhci_hcd *xhci;
1716 	struct xhci_virt_device	*virt_dev;
1717 	int i, ret;
1718 
1719 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
1720 	if (ret <= 0)
1721 		return;
1722 	xhci = hcd_to_xhci(hcd);
1723 
1724 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1725 	virt_dev = xhci->devs[udev->slot_id];
1726 	/* Free any rings allocated for added endpoints */
1727 	for (i = 0; i < 31; ++i) {
1728 		if (virt_dev->eps[i].new_ring) {
1729 			xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
1730 			virt_dev->eps[i].new_ring = NULL;
1731 		}
1732 	}
1733 	xhci_zero_in_ctx(xhci, virt_dev);
1734 }
1735 
xhci_setup_input_ctx_for_config_ep(struct xhci_hcd * xhci,struct xhci_container_ctx * in_ctx,struct xhci_container_ctx * out_ctx,u32 add_flags,u32 drop_flags)1736 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
1737 		struct xhci_container_ctx *in_ctx,
1738 		struct xhci_container_ctx *out_ctx,
1739 		u32 add_flags, u32 drop_flags)
1740 {
1741 	struct xhci_input_control_ctx *ctrl_ctx;
1742 	ctrl_ctx = xhci_get_input_control_ctx(xhci, in_ctx);
1743 	ctrl_ctx->add_flags = add_flags;
1744 	ctrl_ctx->drop_flags = drop_flags;
1745 	xhci_slot_copy(xhci, in_ctx, out_ctx);
1746 	ctrl_ctx->add_flags |= SLOT_FLAG;
1747 
1748 	xhci_dbg(xhci, "Input Context:\n");
1749 	xhci_dbg_ctx(xhci, in_ctx, xhci_last_valid_endpoint(add_flags));
1750 }
1751 
xhci_setup_input_ctx_for_quirk(struct xhci_hcd * xhci,unsigned int slot_id,unsigned int ep_index,struct xhci_dequeue_state * deq_state)1752 static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
1753 		unsigned int slot_id, unsigned int ep_index,
1754 		struct xhci_dequeue_state *deq_state)
1755 {
1756 	struct xhci_container_ctx *in_ctx;
1757 	struct xhci_ep_ctx *ep_ctx;
1758 	u32 added_ctxs;
1759 	dma_addr_t addr;
1760 
1761 	xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
1762 			xhci->devs[slot_id]->out_ctx, ep_index);
1763 	in_ctx = xhci->devs[slot_id]->in_ctx;
1764 	ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
1765 	addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
1766 			deq_state->new_deq_ptr);
1767 	if (addr == 0) {
1768 		xhci_warn(xhci, "WARN Cannot submit config ep after "
1769 				"reset ep command\n");
1770 		xhci_warn(xhci, "WARN deq seg = %p, deq ptr = %p\n",
1771 				deq_state->new_deq_seg,
1772 				deq_state->new_deq_ptr);
1773 		return;
1774 	}
1775 	ep_ctx->deq = addr | deq_state->new_cycle_state;
1776 
1777 	added_ctxs = xhci_get_endpoint_flag_from_index(ep_index);
1778 	xhci_setup_input_ctx_for_config_ep(xhci, xhci->devs[slot_id]->in_ctx,
1779 			xhci->devs[slot_id]->out_ctx, added_ctxs, added_ctxs);
1780 }
1781 
xhci_cleanup_stalled_ring(struct xhci_hcd * xhci,struct usb_device * udev,unsigned int ep_index)1782 void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci,
1783 		struct usb_device *udev, unsigned int ep_index)
1784 {
1785 	struct xhci_dequeue_state deq_state;
1786 	struct xhci_virt_ep *ep;
1787 
1788 	xhci_dbg(xhci, "Cleaning up stalled endpoint ring\n");
1789 	ep = &xhci->devs[udev->slot_id]->eps[ep_index];
1790 	/* We need to move the HW's dequeue pointer past this TD,
1791 	 * or it will attempt to resend it on the next doorbell ring.
1792 	 */
1793 	xhci_find_new_dequeue_state(xhci, udev->slot_id,
1794 			ep_index, ep->stopped_stream, ep->stopped_td,
1795 			&deq_state);
1796 
1797 	/* HW with the reset endpoint quirk will use the saved dequeue state to
1798 	 * issue a configure endpoint command later.
1799 	 */
1800 	if (!(xhci->quirks & XHCI_RESET_EP_QUIRK)) {
1801 		xhci_dbg(xhci, "Queueing new dequeue state\n");
1802 		xhci_queue_new_dequeue_state(xhci, udev->slot_id,
1803 				ep_index, ep->stopped_stream, &deq_state);
1804 	} else {
1805 		/* Better hope no one uses the input context between now and the
1806 		 * reset endpoint completion!
1807 		 * XXX: No idea how this hardware will react when stream rings
1808 		 * are enabled.
1809 		 */
1810 		xhci_dbg(xhci, "Setting up input context for "
1811 				"configure endpoint command\n");
1812 		xhci_setup_input_ctx_for_quirk(xhci, udev->slot_id,
1813 				ep_index, &deq_state);
1814 	}
1815 }
1816 
1817 /* Deal with stalled endpoints.  The core should have sent the control message
1818  * to clear the halt condition.  However, we need to make the xHCI hardware
1819  * reset its sequence number, since a device will expect a sequence number of
1820  * zero after the halt condition is cleared.
1821  * Context: in_interrupt
1822  */
xhci_endpoint_reset(struct usb_hcd * hcd,struct usb_host_endpoint * ep)1823 void xhci_endpoint_reset(struct usb_hcd *hcd,
1824 		struct usb_host_endpoint *ep)
1825 {
1826 	struct xhci_hcd *xhci;
1827 	struct usb_device *udev;
1828 	unsigned int ep_index;
1829 	unsigned long flags;
1830 	int ret;
1831 	struct xhci_virt_ep *virt_ep;
1832 
1833 	xhci = hcd_to_xhci(hcd);
1834 	udev = (struct usb_device *) ep->hcpriv;
1835 	/* Called with a root hub endpoint (or an endpoint that wasn't added
1836 	 * with xhci_add_endpoint()
1837 	 */
1838 	if (!ep->hcpriv)
1839 		return;
1840 	ep_index = xhci_get_endpoint_index(&ep->desc);
1841 	virt_ep = &xhci->devs[udev->slot_id]->eps[ep_index];
1842 	if (!virt_ep->stopped_td) {
1843 		xhci_dbg(xhci, "Endpoint 0x%x not halted, refusing to reset.\n",
1844 				ep->desc.bEndpointAddress);
1845 		return;
1846 	}
1847 	if (usb_endpoint_xfer_control(&ep->desc)) {
1848 		xhci_dbg(xhci, "Control endpoint stall already handled.\n");
1849 		return;
1850 	}
1851 
1852 	xhci_dbg(xhci, "Queueing reset endpoint command\n");
1853 	spin_lock_irqsave(&xhci->lock, flags);
1854 	ret = xhci_queue_reset_ep(xhci, udev->slot_id, ep_index);
1855 	/*
1856 	 * Can't change the ring dequeue pointer until it's transitioned to the
1857 	 * stopped state, which is only upon a successful reset endpoint
1858 	 * command.  Better hope that last command worked!
1859 	 */
1860 	if (!ret) {
1861 		xhci_cleanup_stalled_ring(xhci, udev, ep_index);
1862 		kfree(virt_ep->stopped_td);
1863 		xhci_ring_cmd_db(xhci);
1864 	}
1865 	virt_ep->stopped_td = NULL;
1866 	virt_ep->stopped_trb = NULL;
1867 	virt_ep->stopped_stream = 0;
1868 	spin_unlock_irqrestore(&xhci->lock, flags);
1869 
1870 	if (ret)
1871 		xhci_warn(xhci, "FIXME allocate a new ring segment\n");
1872 }
1873 
xhci_check_streams_endpoint(struct xhci_hcd * xhci,struct usb_device * udev,struct usb_host_endpoint * ep,unsigned int slot_id)1874 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
1875 		struct usb_device *udev, struct usb_host_endpoint *ep,
1876 		unsigned int slot_id)
1877 {
1878 	int ret;
1879 	unsigned int ep_index;
1880 	unsigned int ep_state;
1881 
1882 	if (!ep)
1883 		return -EINVAL;
1884 	ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
1885 	if (ret <= 0)
1886 		return -EINVAL;
1887 	if (ep->ss_ep_comp.bmAttributes == 0) {
1888 		xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
1889 				" descriptor for ep 0x%x does not support streams\n",
1890 				ep->desc.bEndpointAddress);
1891 		return -EINVAL;
1892 	}
1893 
1894 	ep_index = xhci_get_endpoint_index(&ep->desc);
1895 	ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
1896 	if (ep_state & EP_HAS_STREAMS ||
1897 			ep_state & EP_GETTING_STREAMS) {
1898 		xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
1899 				"already has streams set up.\n",
1900 				ep->desc.bEndpointAddress);
1901 		xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
1902 				"dynamic stream context array reallocation.\n");
1903 		return -EINVAL;
1904 	}
1905 	if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
1906 		xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
1907 				"endpoint 0x%x; URBs are pending.\n",
1908 				ep->desc.bEndpointAddress);
1909 		return -EINVAL;
1910 	}
1911 	return 0;
1912 }
1913 
xhci_calculate_streams_entries(struct xhci_hcd * xhci,unsigned int * num_streams,unsigned int * num_stream_ctxs)1914 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
1915 		unsigned int *num_streams, unsigned int *num_stream_ctxs)
1916 {
1917 	unsigned int max_streams;
1918 
1919 	/* The stream context array size must be a power of two */
1920 	*num_stream_ctxs = roundup_pow_of_two(*num_streams);
1921 	/*
1922 	 * Find out how many primary stream array entries the host controller
1923 	 * supports.  Later we may use secondary stream arrays (similar to 2nd
1924 	 * level page entries), but that's an optional feature for xHCI host
1925 	 * controllers. xHCs must support at least 4 stream IDs.
1926 	 */
1927 	max_streams = HCC_MAX_PSA(xhci->hcc_params);
1928 	if (*num_stream_ctxs > max_streams) {
1929 		xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
1930 				max_streams);
1931 		*num_stream_ctxs = max_streams;
1932 		*num_streams = max_streams;
1933 	}
1934 }
1935 
1936 /* Returns an error code if one of the endpoint already has streams.
1937  * This does not change any data structures, it only checks and gathers
1938  * information.
1939  */
xhci_calculate_streams_and_bitmask(struct xhci_hcd * xhci,struct usb_device * udev,struct usb_host_endpoint ** eps,unsigned int num_eps,unsigned int * num_streams,u32 * changed_ep_bitmask)1940 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
1941 		struct usb_device *udev,
1942 		struct usb_host_endpoint **eps, unsigned int num_eps,
1943 		unsigned int *num_streams, u32 *changed_ep_bitmask)
1944 {
1945 	unsigned int max_streams;
1946 	unsigned int endpoint_flag;
1947 	int i;
1948 	int ret;
1949 
1950 	for (i = 0; i < num_eps; i++) {
1951 		ret = xhci_check_streams_endpoint(xhci, udev,
1952 				eps[i], udev->slot_id);
1953 		if (ret < 0)
1954 			return ret;
1955 
1956 		max_streams = USB_SS_MAX_STREAMS(
1957 				eps[i]->ss_ep_comp.bmAttributes);
1958 		if (max_streams < (*num_streams - 1)) {
1959 			xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
1960 					eps[i]->desc.bEndpointAddress,
1961 					max_streams);
1962 			*num_streams = max_streams+1;
1963 		}
1964 
1965 		endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
1966 		if (*changed_ep_bitmask & endpoint_flag)
1967 			return -EINVAL;
1968 		*changed_ep_bitmask |= endpoint_flag;
1969 	}
1970 	return 0;
1971 }
1972 
xhci_calculate_no_streams_bitmask(struct xhci_hcd * xhci,struct usb_device * udev,struct usb_host_endpoint ** eps,unsigned int num_eps)1973 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
1974 		struct usb_device *udev,
1975 		struct usb_host_endpoint **eps, unsigned int num_eps)
1976 {
1977 	u32 changed_ep_bitmask = 0;
1978 	unsigned int slot_id;
1979 	unsigned int ep_index;
1980 	unsigned int ep_state;
1981 	int i;
1982 
1983 	slot_id = udev->slot_id;
1984 	if (!xhci->devs[slot_id])
1985 		return 0;
1986 
1987 	for (i = 0; i < num_eps; i++) {
1988 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
1989 		ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
1990 		/* Are streams already being freed for the endpoint? */
1991 		if (ep_state & EP_GETTING_NO_STREAMS) {
1992 			xhci_warn(xhci, "WARN Can't disable streams for "
1993 					"endpoint 0x%x\n, "
1994 					"streams are being disabled already.",
1995 					eps[i]->desc.bEndpointAddress);
1996 			return 0;
1997 		}
1998 		/* Are there actually any streams to free? */
1999 		if (!(ep_state & EP_HAS_STREAMS) &&
2000 				!(ep_state & EP_GETTING_STREAMS)) {
2001 			xhci_warn(xhci, "WARN Can't disable streams for "
2002 					"endpoint 0x%x\n, "
2003 					"streams are already disabled!",
2004 					eps[i]->desc.bEndpointAddress);
2005 			xhci_warn(xhci, "WARN xhci_free_streams() called "
2006 					"with non-streams endpoint\n");
2007 			return 0;
2008 		}
2009 		changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
2010 	}
2011 	return changed_ep_bitmask;
2012 }
2013 
2014 /*
2015  * The USB device drivers use this function (though the HCD interface in USB
2016  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
2017  * coordinate mass storage command queueing across multiple endpoints (basically
2018  * a stream ID == a task ID).
2019  *
2020  * Setting up streams involves allocating the same size stream context array
2021  * for each endpoint and issuing a configure endpoint command for all endpoints.
2022  *
2023  * Don't allow the call to succeed if one endpoint only supports one stream
2024  * (which means it doesn't support streams at all).
2025  *
2026  * Drivers may get less stream IDs than they asked for, if the host controller
2027  * hardware or endpoints claim they can't support the number of requested
2028  * stream IDs.
2029  */
xhci_alloc_streams(struct usb_hcd * hcd,struct usb_device * udev,struct usb_host_endpoint ** eps,unsigned int num_eps,unsigned int num_streams,gfp_t mem_flags)2030 int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
2031 		struct usb_host_endpoint **eps, unsigned int num_eps,
2032 		unsigned int num_streams, gfp_t mem_flags)
2033 {
2034 	int i, ret;
2035 	struct xhci_hcd *xhci;
2036 	struct xhci_virt_device *vdev;
2037 	struct xhci_command *config_cmd;
2038 	unsigned int ep_index;
2039 	unsigned int num_stream_ctxs;
2040 	unsigned long flags;
2041 	u32 changed_ep_bitmask = 0;
2042 
2043 	if (!eps)
2044 		return -EINVAL;
2045 
2046 	/* Add one to the number of streams requested to account for
2047 	 * stream 0 that is reserved for xHCI usage.
2048 	 */
2049 	num_streams += 1;
2050 	xhci = hcd_to_xhci(hcd);
2051 	xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
2052 			num_streams);
2053 
2054 	config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
2055 	if (!config_cmd) {
2056 		xhci_dbg(xhci, "Could not allocate xHCI command structure.\n");
2057 		return -ENOMEM;
2058 	}
2059 
2060 	/* Check to make sure all endpoints are not already configured for
2061 	 * streams.  While we're at it, find the maximum number of streams that
2062 	 * all the endpoints will support and check for duplicate endpoints.
2063 	 */
2064 	spin_lock_irqsave(&xhci->lock, flags);
2065 	ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
2066 			num_eps, &num_streams, &changed_ep_bitmask);
2067 	if (ret < 0) {
2068 		xhci_free_command(xhci, config_cmd);
2069 		spin_unlock_irqrestore(&xhci->lock, flags);
2070 		return ret;
2071 	}
2072 	if (num_streams <= 1) {
2073 		xhci_warn(xhci, "WARN: endpoints can't handle "
2074 				"more than one stream.\n");
2075 		xhci_free_command(xhci, config_cmd);
2076 		spin_unlock_irqrestore(&xhci->lock, flags);
2077 		return -EINVAL;
2078 	}
2079 	vdev = xhci->devs[udev->slot_id];
2080 	/* Mark each endpoint as being in transition, so
2081 	 * xhci_urb_enqueue() will reject all URBs.
2082 	 */
2083 	for (i = 0; i < num_eps; i++) {
2084 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
2085 		vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
2086 	}
2087 	spin_unlock_irqrestore(&xhci->lock, flags);
2088 
2089 	/* Setup internal data structures and allocate HW data structures for
2090 	 * streams (but don't install the HW structures in the input context
2091 	 * until we're sure all memory allocation succeeded).
2092 	 */
2093 	xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
2094 	xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
2095 			num_stream_ctxs, num_streams);
2096 
2097 	for (i = 0; i < num_eps; i++) {
2098 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
2099 		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
2100 				num_stream_ctxs,
2101 				num_streams, mem_flags);
2102 		if (!vdev->eps[ep_index].stream_info)
2103 			goto cleanup;
2104 		/* Set maxPstreams in endpoint context and update deq ptr to
2105 		 * point to stream context array. FIXME
2106 		 */
2107 	}
2108 
2109 	/* Set up the input context for a configure endpoint command. */
2110 	for (i = 0; i < num_eps; i++) {
2111 		struct xhci_ep_ctx *ep_ctx;
2112 
2113 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
2114 		ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
2115 
2116 		xhci_endpoint_copy(xhci, config_cmd->in_ctx,
2117 				vdev->out_ctx, ep_index);
2118 		xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
2119 				vdev->eps[ep_index].stream_info);
2120 	}
2121 	/* Tell the HW to drop its old copy of the endpoint context info
2122 	 * and add the updated copy from the input context.
2123 	 */
2124 	xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
2125 			vdev->out_ctx, changed_ep_bitmask, changed_ep_bitmask);
2126 
2127 	/* Issue and wait for the configure endpoint command */
2128 	ret = xhci_configure_endpoint(xhci, udev, config_cmd,
2129 			false, false);
2130 
2131 	/* xHC rejected the configure endpoint command for some reason, so we
2132 	 * leave the old ring intact and free our internal streams data
2133 	 * structure.
2134 	 */
2135 	if (ret < 0)
2136 		goto cleanup;
2137 
2138 	spin_lock_irqsave(&xhci->lock, flags);
2139 	for (i = 0; i < num_eps; i++) {
2140 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
2141 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
2142 		xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
2143 			 udev->slot_id, ep_index);
2144 		vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
2145 	}
2146 	xhci_free_command(xhci, config_cmd);
2147 	spin_unlock_irqrestore(&xhci->lock, flags);
2148 
2149 	/* Subtract 1 for stream 0, which drivers can't use */
2150 	return num_streams - 1;
2151 
2152 cleanup:
2153 	/* If it didn't work, free the streams! */
2154 	for (i = 0; i < num_eps; i++) {
2155 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
2156 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
2157 		vdev->eps[ep_index].stream_info = NULL;
2158 		/* FIXME Unset maxPstreams in endpoint context and
2159 		 * update deq ptr to point to normal string ring.
2160 		 */
2161 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
2162 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
2163 		xhci_endpoint_zero(xhci, vdev, eps[i]);
2164 	}
2165 	xhci_free_command(xhci, config_cmd);
2166 	return -ENOMEM;
2167 }
2168 
2169 /* Transition the endpoint from using streams to being a "normal" endpoint
2170  * without streams.
2171  *
2172  * Modify the endpoint context state, submit a configure endpoint command,
2173  * and free all endpoint rings for streams if that completes successfully.
2174  */
xhci_free_streams(struct usb_hcd * hcd,struct usb_device * udev,struct usb_host_endpoint ** eps,unsigned int num_eps,gfp_t mem_flags)2175 int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
2176 		struct usb_host_endpoint **eps, unsigned int num_eps,
2177 		gfp_t mem_flags)
2178 {
2179 	int i, ret;
2180 	struct xhci_hcd *xhci;
2181 	struct xhci_virt_device *vdev;
2182 	struct xhci_command *command;
2183 	unsigned int ep_index;
2184 	unsigned long flags;
2185 	u32 changed_ep_bitmask;
2186 
2187 	xhci = hcd_to_xhci(hcd);
2188 	vdev = xhci->devs[udev->slot_id];
2189 
2190 	/* Set up a configure endpoint command to remove the streams rings */
2191 	spin_lock_irqsave(&xhci->lock, flags);
2192 	changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
2193 			udev, eps, num_eps);
2194 	if (changed_ep_bitmask == 0) {
2195 		spin_unlock_irqrestore(&xhci->lock, flags);
2196 		return -EINVAL;
2197 	}
2198 
2199 	/* Use the xhci_command structure from the first endpoint.  We may have
2200 	 * allocated too many, but the driver may call xhci_free_streams() for
2201 	 * each endpoint it grouped into one call to xhci_alloc_streams().
2202 	 */
2203 	ep_index = xhci_get_endpoint_index(&eps[0]->desc);
2204 	command = vdev->eps[ep_index].stream_info->free_streams_command;
2205 	for (i = 0; i < num_eps; i++) {
2206 		struct xhci_ep_ctx *ep_ctx;
2207 
2208 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
2209 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
2210 		xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
2211 			EP_GETTING_NO_STREAMS;
2212 
2213 		xhci_endpoint_copy(xhci, command->in_ctx,
2214 				vdev->out_ctx, ep_index);
2215 		xhci_setup_no_streams_ep_input_ctx(xhci, ep_ctx,
2216 				&vdev->eps[ep_index]);
2217 	}
2218 	xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
2219 			vdev->out_ctx, changed_ep_bitmask, changed_ep_bitmask);
2220 	spin_unlock_irqrestore(&xhci->lock, flags);
2221 
2222 	/* Issue and wait for the configure endpoint command,
2223 	 * which must succeed.
2224 	 */
2225 	ret = xhci_configure_endpoint(xhci, udev, command,
2226 			false, true);
2227 
2228 	/* xHC rejected the configure endpoint command for some reason, so we
2229 	 * leave the streams rings intact.
2230 	 */
2231 	if (ret < 0)
2232 		return ret;
2233 
2234 	spin_lock_irqsave(&xhci->lock, flags);
2235 	for (i = 0; i < num_eps; i++) {
2236 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
2237 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
2238 		vdev->eps[ep_index].stream_info = NULL;
2239 		/* FIXME Unset maxPstreams in endpoint context and
2240 		 * update deq ptr to point to normal string ring.
2241 		 */
2242 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
2243 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
2244 	}
2245 	spin_unlock_irqrestore(&xhci->lock, flags);
2246 
2247 	return 0;
2248 }
2249 
2250 /*
2251  * This submits a Reset Device Command, which will set the device state to 0,
2252  * set the device address to 0, and disable all the endpoints except the default
2253  * control endpoint.  The USB core should come back and call
2254  * xhci_address_device(), and then re-set up the configuration.  If this is
2255  * called because of a usb_reset_and_verify_device(), then the old alternate
2256  * settings will be re-installed through the normal bandwidth allocation
2257  * functions.
2258  *
2259  * Wait for the Reset Device command to finish.  Remove all structures
2260  * associated with the endpoints that were disabled.  Clear the input device
2261  * structure?  Cache the rings?  Reset the control endpoint 0 max packet size?
2262  *
2263  * If the virt_dev to be reset does not exist or does not match the udev,
2264  * it means the device is lost, possibly due to the xHC restore error and
2265  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
2266  * re-allocate the device.
2267  */
xhci_discover_or_reset_device(struct usb_hcd * hcd,struct usb_device * udev)2268 int xhci_discover_or_reset_device(struct usb_hcd *hcd, struct usb_device *udev)
2269 {
2270 	int ret, i;
2271 	unsigned long flags;
2272 	struct xhci_hcd *xhci;
2273 	unsigned int slot_id;
2274 	struct xhci_virt_device *virt_dev;
2275 	struct xhci_command *reset_device_cmd;
2276 	int timeleft;
2277 	int last_freed_endpoint;
2278 
2279 	ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
2280 	if (ret <= 0)
2281 		return ret;
2282 	xhci = hcd_to_xhci(hcd);
2283 	slot_id = udev->slot_id;
2284 	virt_dev = xhci->devs[slot_id];
2285 	if (!virt_dev) {
2286 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
2287 				"not exist. Re-allocate the device\n", slot_id);
2288 		ret = xhci_alloc_dev(hcd, udev);
2289 		if (ret == 1)
2290 			return 0;
2291 		else
2292 			return -EINVAL;
2293 	}
2294 
2295 	if (virt_dev->udev != udev) {
2296 		/* If the virt_dev and the udev does not match, this virt_dev
2297 		 * may belong to another udev.
2298 		 * Re-allocate the device.
2299 		 */
2300 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
2301 				"not match the udev. Re-allocate the device\n",
2302 				slot_id);
2303 		ret = xhci_alloc_dev(hcd, udev);
2304 		if (ret == 1)
2305 			return 0;
2306 		else
2307 			return -EINVAL;
2308 	}
2309 
2310 	xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
2311 	/* Allocate the command structure that holds the struct completion.
2312 	 * Assume we're in process context, since the normal device reset
2313 	 * process has to wait for the device anyway.  Storage devices are
2314 	 * reset as part of error handling, so use GFP_NOIO instead of
2315 	 * GFP_KERNEL.
2316 	 */
2317 	reset_device_cmd = xhci_alloc_command(xhci, false, true, GFP_NOIO);
2318 	if (!reset_device_cmd) {
2319 		xhci_dbg(xhci, "Couldn't allocate command structure.\n");
2320 		return -ENOMEM;
2321 	}
2322 
2323 	/* Attempt to submit the Reset Device command to the command ring */
2324 	spin_lock_irqsave(&xhci->lock, flags);
2325 	reset_device_cmd->command_trb = xhci->cmd_ring->enqueue;
2326 
2327 	/* Enqueue pointer can be left pointing to the link TRB,
2328 	 * we must handle that
2329 	 */
2330 	if ((reset_device_cmd->command_trb->link.control & TRB_TYPE_BITMASK)
2331 			== TRB_TYPE(TRB_LINK))
2332 		reset_device_cmd->command_trb =
2333 			xhci->cmd_ring->enq_seg->next->trbs;
2334 
2335 	list_add_tail(&reset_device_cmd->cmd_list, &virt_dev->cmd_list);
2336 	ret = xhci_queue_reset_device(xhci, slot_id);
2337 	if (ret) {
2338 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
2339 		list_del(&reset_device_cmd->cmd_list);
2340 		spin_unlock_irqrestore(&xhci->lock, flags);
2341 		goto command_cleanup;
2342 	}
2343 	xhci_ring_cmd_db(xhci);
2344 	spin_unlock_irqrestore(&xhci->lock, flags);
2345 
2346 	/* Wait for the Reset Device command to finish */
2347 	timeleft = wait_for_completion_interruptible_timeout(
2348 			reset_device_cmd->completion,
2349 			USB_CTRL_SET_TIMEOUT);
2350 	if (timeleft <= 0) {
2351 		xhci_warn(xhci, "%s while waiting for reset device command\n",
2352 				timeleft == 0 ? "Timeout" : "Signal");
2353 		spin_lock_irqsave(&xhci->lock, flags);
2354 		/* The timeout might have raced with the event ring handler, so
2355 		 * only delete from the list if the item isn't poisoned.
2356 		 */
2357 		if (reset_device_cmd->cmd_list.next != LIST_POISON1)
2358 			list_del(&reset_device_cmd->cmd_list);
2359 		spin_unlock_irqrestore(&xhci->lock, flags);
2360 		ret = -ETIME;
2361 		goto command_cleanup;
2362 	}
2363 
2364 	/* The Reset Device command can't fail, according to the 0.95/0.96 spec,
2365 	 * unless we tried to reset a slot ID that wasn't enabled,
2366 	 * or the device wasn't in the addressed or configured state.
2367 	 */
2368 	ret = reset_device_cmd->status;
2369 	switch (ret) {
2370 	case COMP_EBADSLT: /* 0.95 completion code for bad slot ID */
2371 	case COMP_CTX_STATE: /* 0.96 completion code for same thing */
2372 		xhci_info(xhci, "Can't reset device (slot ID %u) in %s state\n",
2373 				slot_id,
2374 				xhci_get_slot_state(xhci, virt_dev->out_ctx));
2375 		xhci_info(xhci, "Not freeing device rings.\n");
2376 		/* Don't treat this as an error.  May change my mind later. */
2377 		ret = 0;
2378 		goto command_cleanup;
2379 	case COMP_SUCCESS:
2380 		xhci_dbg(xhci, "Successful reset device command.\n");
2381 		break;
2382 	default:
2383 		if (xhci_is_vendor_info_code(xhci, ret))
2384 			break;
2385 		xhci_warn(xhci, "Unknown completion code %u for "
2386 				"reset device command.\n", ret);
2387 		ret = -EINVAL;
2388 		goto command_cleanup;
2389 	}
2390 
2391 	/* Everything but endpoint 0 is disabled, so free or cache the rings. */
2392 	last_freed_endpoint = 1;
2393 	for (i = 1; i < 31; ++i) {
2394 		struct xhci_virt_ep *ep = &virt_dev->eps[i];
2395 
2396 		if (ep->ep_state & EP_HAS_STREAMS) {
2397 			xhci_free_stream_info(xhci, ep->stream_info);
2398 			ep->stream_info = NULL;
2399 			ep->ep_state &= ~EP_HAS_STREAMS;
2400 		}
2401 
2402 		if (ep->ring) {
2403 			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
2404 			last_freed_endpoint = i;
2405 		}
2406 	}
2407 	xhci_dbg(xhci, "Output context after successful reset device cmd:\n");
2408 	xhci_dbg_ctx(xhci, virt_dev->out_ctx, last_freed_endpoint);
2409 	ret = 0;
2410 
2411 command_cleanup:
2412 	xhci_free_command(xhci, reset_device_cmd);
2413 	return ret;
2414 }
2415 
2416 /*
2417  * At this point, the struct usb_device is about to go away, the device has
2418  * disconnected, and all traffic has been stopped and the endpoints have been
2419  * disabled.  Free any HC data structures associated with that device.
2420  */
xhci_free_dev(struct usb_hcd * hcd,struct usb_device * udev)2421 void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
2422 {
2423 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2424 	struct xhci_virt_device *virt_dev;
2425 	unsigned long flags;
2426 	u32 state;
2427 	int i, ret;
2428 
2429 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2430 	if (ret <= 0)
2431 		return;
2432 
2433 	virt_dev = xhci->devs[udev->slot_id];
2434 
2435 	/* Stop any wayward timer functions (which may grab the lock) */
2436 	for (i = 0; i < 31; ++i) {
2437 		virt_dev->eps[i].ep_state &= ~EP_HALT_PENDING;
2438 		del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
2439 	}
2440 
2441 	spin_lock_irqsave(&xhci->lock, flags);
2442 	/* Don't disable the slot if the host controller is dead. */
2443 	state = xhci_readl(xhci, &xhci->op_regs->status);
2444 	if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING)) {
2445 		xhci_free_virt_device(xhci, udev->slot_id);
2446 		spin_unlock_irqrestore(&xhci->lock, flags);
2447 		return;
2448 	}
2449 
2450 	if (xhci_queue_slot_control(xhci, TRB_DISABLE_SLOT, udev->slot_id)) {
2451 		spin_unlock_irqrestore(&xhci->lock, flags);
2452 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
2453 		return;
2454 	}
2455 	xhci_ring_cmd_db(xhci);
2456 	spin_unlock_irqrestore(&xhci->lock, flags);
2457 	/*
2458 	 * Event command completion handler will free any data structures
2459 	 * associated with the slot.  XXX Can free sleep?
2460 	 */
2461 }
2462 
2463 /*
2464  * Returns 0 if the xHC ran out of device slots, the Enable Slot command
2465  * timed out, or allocating memory failed.  Returns 1 on success.
2466  */
xhci_alloc_dev(struct usb_hcd * hcd,struct usb_device * udev)2467 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
2468 {
2469 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2470 	unsigned long flags;
2471 	int timeleft;
2472 	int ret;
2473 
2474 	spin_lock_irqsave(&xhci->lock, flags);
2475 	ret = xhci_queue_slot_control(xhci, TRB_ENABLE_SLOT, 0);
2476 	if (ret) {
2477 		spin_unlock_irqrestore(&xhci->lock, flags);
2478 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
2479 		return 0;
2480 	}
2481 	xhci_ring_cmd_db(xhci);
2482 	spin_unlock_irqrestore(&xhci->lock, flags);
2483 
2484 	/* XXX: how much time for xHC slot assignment? */
2485 	timeleft = wait_for_completion_interruptible_timeout(&xhci->addr_dev,
2486 			USB_CTRL_SET_TIMEOUT);
2487 	if (timeleft <= 0) {
2488 		xhci_warn(xhci, "%s while waiting for a slot\n",
2489 				timeleft == 0 ? "Timeout" : "Signal");
2490 		/* FIXME cancel the enable slot request */
2491 		return 0;
2492 	}
2493 
2494 	if (!xhci->slot_id) {
2495 		xhci_err(xhci, "Error while assigning device slot ID\n");
2496 		return 0;
2497 	}
2498 	/* xhci_alloc_virt_device() does not touch rings; no need to lock.
2499 	 * Use GFP_NOIO, since this function can be called from
2500 	 * xhci_discover_or_reset_device(), which may be called as part of
2501 	 * mass storage driver error handling.
2502 	 */
2503 	if (!xhci_alloc_virt_device(xhci, xhci->slot_id, udev, GFP_NOIO)) {
2504 		/* Disable slot, if we can do it without mem alloc */
2505 		xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
2506 		spin_lock_irqsave(&xhci->lock, flags);
2507 		if (!xhci_queue_slot_control(xhci, TRB_DISABLE_SLOT, udev->slot_id))
2508 			xhci_ring_cmd_db(xhci);
2509 		spin_unlock_irqrestore(&xhci->lock, flags);
2510 		return 0;
2511 	}
2512 	udev->slot_id = xhci->slot_id;
2513 	/* Is this a LS or FS device under a HS hub? */
2514 	/* Hub or peripherial? */
2515 	return 1;
2516 }
2517 
2518 /*
2519  * Issue an Address Device command (which will issue a SetAddress request to
2520  * the device).
2521  * We should be protected by the usb_address0_mutex in khubd's hub_port_init, so
2522  * we should only issue and wait on one address command at the same time.
2523  *
2524  * We add one to the device address issued by the hardware because the USB core
2525  * uses address 1 for the root hubs (even though they're not really devices).
2526  */
xhci_address_device(struct usb_hcd * hcd,struct usb_device * udev)2527 int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
2528 {
2529 	unsigned long flags;
2530 	int timeleft;
2531 	struct xhci_virt_device *virt_dev;
2532 	int ret = 0;
2533 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2534 	struct xhci_slot_ctx *slot_ctx;
2535 	struct xhci_input_control_ctx *ctrl_ctx;
2536 	u64 temp_64;
2537 
2538 	if (!udev->slot_id) {
2539 		xhci_dbg(xhci, "Bad Slot ID %d\n", udev->slot_id);
2540 		return -EINVAL;
2541 	}
2542 
2543 	virt_dev = xhci->devs[udev->slot_id];
2544 
2545 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2546 	/*
2547 	 * If this is the first Set Address since device plug-in or
2548 	 * virt_device realloaction after a resume with an xHCI power loss,
2549 	 * then set up the slot context.
2550 	 */
2551 	if (!slot_ctx->dev_info)
2552 		xhci_setup_addressable_virt_dev(xhci, udev);
2553 	/* Otherwise, update the control endpoint ring enqueue pointer. */
2554 	else
2555 		xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
2556 	xhci_dbg(xhci, "Slot ID %d Input Context:\n", udev->slot_id);
2557 	xhci_dbg_ctx(xhci, virt_dev->in_ctx, 2);
2558 
2559 	spin_lock_irqsave(&xhci->lock, flags);
2560 	ret = xhci_queue_address_device(xhci, virt_dev->in_ctx->dma,
2561 					udev->slot_id);
2562 	if (ret) {
2563 		spin_unlock_irqrestore(&xhci->lock, flags);
2564 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
2565 		return ret;
2566 	}
2567 	xhci_ring_cmd_db(xhci);
2568 	spin_unlock_irqrestore(&xhci->lock, flags);
2569 
2570 	/* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
2571 	timeleft = wait_for_completion_interruptible_timeout(&xhci->addr_dev,
2572 			USB_CTRL_SET_TIMEOUT);
2573 	/* FIXME: From section 4.3.4: "Software shall be responsible for timing
2574 	 * the SetAddress() "recovery interval" required by USB and aborting the
2575 	 * command on a timeout.
2576 	 */
2577 	if (timeleft <= 0) {
2578 		xhci_warn(xhci, "%s while waiting for a slot\n",
2579 				timeleft == 0 ? "Timeout" : "Signal");
2580 		/* FIXME cancel the address device command */
2581 		return -ETIME;
2582 	}
2583 
2584 	switch (virt_dev->cmd_status) {
2585 	case COMP_CTX_STATE:
2586 	case COMP_EBADSLT:
2587 		xhci_err(xhci, "Setup ERROR: address device command for slot %d.\n",
2588 				udev->slot_id);
2589 		ret = -EINVAL;
2590 		break;
2591 	case COMP_TX_ERR:
2592 		dev_warn(&udev->dev, "Device not responding to set address.\n");
2593 		ret = -EPROTO;
2594 		break;
2595 	case COMP_SUCCESS:
2596 		xhci_dbg(xhci, "Successful Address Device command\n");
2597 		break;
2598 	default:
2599 		xhci_err(xhci, "ERROR: unexpected command completion "
2600 				"code 0x%x.\n", virt_dev->cmd_status);
2601 		xhci_dbg(xhci, "Slot ID %d Output Context:\n", udev->slot_id);
2602 		xhci_dbg_ctx(xhci, virt_dev->out_ctx, 2);
2603 		ret = -EINVAL;
2604 		break;
2605 	}
2606 	if (ret) {
2607 		return ret;
2608 	}
2609 	temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
2610 	xhci_dbg(xhci, "Op regs DCBAA ptr = %#016llx\n", temp_64);
2611 	xhci_dbg(xhci, "Slot ID %d dcbaa entry @%p = %#016llx\n",
2612 			udev->slot_id,
2613 			&xhci->dcbaa->dev_context_ptrs[udev->slot_id],
2614 			(unsigned long long)
2615 				xhci->dcbaa->dev_context_ptrs[udev->slot_id]);
2616 	xhci_dbg(xhci, "Output Context DMA address = %#08llx\n",
2617 			(unsigned long long)virt_dev->out_ctx->dma);
2618 	xhci_dbg(xhci, "Slot ID %d Input Context:\n", udev->slot_id);
2619 	xhci_dbg_ctx(xhci, virt_dev->in_ctx, 2);
2620 	xhci_dbg(xhci, "Slot ID %d Output Context:\n", udev->slot_id);
2621 	xhci_dbg_ctx(xhci, virt_dev->out_ctx, 2);
2622 	/*
2623 	 * USB core uses address 1 for the roothubs, so we add one to the
2624 	 * address given back to us by the HC.
2625 	 */
2626 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
2627 	/* Use kernel assigned address for devices; store xHC assigned
2628 	 * address locally. */
2629 	virt_dev->address = (slot_ctx->dev_state & DEV_ADDR_MASK) + 1;
2630 	/* Zero the input context control for later use */
2631 	ctrl_ctx = xhci_get_input_control_ctx(xhci, virt_dev->in_ctx);
2632 	ctrl_ctx->add_flags = 0;
2633 	ctrl_ctx->drop_flags = 0;
2634 
2635 	xhci_dbg(xhci, "Internal device address = %d\n", virt_dev->address);
2636 
2637 	return 0;
2638 }
2639 
2640 /* Once a hub descriptor is fetched for a device, we need to update the xHC's
2641  * internal data structures for the device.
2642  */
xhci_update_hub_device(struct usb_hcd * hcd,struct usb_device * hdev,struct usb_tt * tt,gfp_t mem_flags)2643 int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
2644 			struct usb_tt *tt, gfp_t mem_flags)
2645 {
2646 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2647 	struct xhci_virt_device *vdev;
2648 	struct xhci_command *config_cmd;
2649 	struct xhci_input_control_ctx *ctrl_ctx;
2650 	struct xhci_slot_ctx *slot_ctx;
2651 	unsigned long flags;
2652 	unsigned think_time;
2653 	int ret;
2654 
2655 	/* Ignore root hubs */
2656 	if (!hdev->parent)
2657 		return 0;
2658 
2659 	vdev = xhci->devs[hdev->slot_id];
2660 	if (!vdev) {
2661 		xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
2662 		return -EINVAL;
2663 	}
2664 	config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
2665 	if (!config_cmd) {
2666 		xhci_dbg(xhci, "Could not allocate xHCI command structure.\n");
2667 		return -ENOMEM;
2668 	}
2669 
2670 	spin_lock_irqsave(&xhci->lock, flags);
2671 	xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
2672 	ctrl_ctx = xhci_get_input_control_ctx(xhci, config_cmd->in_ctx);
2673 	ctrl_ctx->add_flags |= SLOT_FLAG;
2674 	slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
2675 	slot_ctx->dev_info |= DEV_HUB;
2676 	if (tt->multi)
2677 		slot_ctx->dev_info |= DEV_MTT;
2678 	if (xhci->hci_version > 0x95) {
2679 		xhci_dbg(xhci, "xHCI version %x needs hub "
2680 				"TT think time and number of ports\n",
2681 				(unsigned int) xhci->hci_version);
2682 		slot_ctx->dev_info2 |= XHCI_MAX_PORTS(hdev->maxchild);
2683 		/* Set TT think time - convert from ns to FS bit times.
2684 		 * 0 = 8 FS bit times, 1 = 16 FS bit times,
2685 		 * 2 = 24 FS bit times, 3 = 32 FS bit times.
2686 		 */
2687 		think_time = tt->think_time;
2688 		if (think_time != 0)
2689 			think_time = (think_time / 666) - 1;
2690 		slot_ctx->tt_info |= TT_THINK_TIME(think_time);
2691 	} else {
2692 		xhci_dbg(xhci, "xHCI version %x doesn't need hub "
2693 				"TT think time or number of ports\n",
2694 				(unsigned int) xhci->hci_version);
2695 	}
2696 	slot_ctx->dev_state = 0;
2697 	spin_unlock_irqrestore(&xhci->lock, flags);
2698 
2699 	xhci_dbg(xhci, "Set up %s for hub device.\n",
2700 			(xhci->hci_version > 0x95) ?
2701 			"configure endpoint" : "evaluate context");
2702 	xhci_dbg(xhci, "Slot %u Input Context:\n", hdev->slot_id);
2703 	xhci_dbg_ctx(xhci, config_cmd->in_ctx, 0);
2704 
2705 	/* Issue and wait for the configure endpoint or
2706 	 * evaluate context command.
2707 	 */
2708 	if (xhci->hci_version > 0x95)
2709 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
2710 				false, false);
2711 	else
2712 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
2713 				true, false);
2714 
2715 	xhci_dbg(xhci, "Slot %u Output Context:\n", hdev->slot_id);
2716 	xhci_dbg_ctx(xhci, vdev->out_ctx, 0);
2717 
2718 	xhci_free_command(xhci, config_cmd);
2719 	return ret;
2720 }
2721 
xhci_get_frame(struct usb_hcd * hcd)2722 int xhci_get_frame(struct usb_hcd *hcd)
2723 {
2724 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2725 	/* EHCI mods by the periodic size.  Why? */
2726 	return xhci_readl(xhci, &xhci->run_regs->microframe_index) >> 3;
2727 }
2728 
2729 MODULE_DESCRIPTION(DRIVER_DESC);
2730 MODULE_AUTHOR(DRIVER_AUTHOR);
2731 MODULE_LICENSE("GPL");
2732 
xhci_hcd_init(void)2733 static int __init xhci_hcd_init(void)
2734 {
2735 #ifdef CONFIG_PCI
2736 	int retval = 0;
2737 
2738 	retval = xhci_register_pci();
2739 
2740 	if (retval < 0) {
2741 		printk(KERN_DEBUG "Problem registering PCI driver.");
2742 		return retval;
2743 	}
2744 #endif
2745 	/*
2746 	 * Check the compiler generated sizes of structures that must be laid
2747 	 * out in specific ways for hardware access.
2748 	 */
2749 	BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
2750 	BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
2751 	BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
2752 	/* xhci_device_control has eight fields, and also
2753 	 * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
2754 	 */
2755 	BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
2756 	BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
2757 	BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
2758 	BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 7*32/8);
2759 	BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
2760 	/* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
2761 	BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
2762 	BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
2763 	return 0;
2764 }
2765 module_init(xhci_hcd_init);
2766 
xhci_hcd_cleanup(void)2767 static void __exit xhci_hcd_cleanup(void)
2768 {
2769 #ifdef CONFIG_PCI
2770 	xhci_unregister_pci();
2771 #endif
2772 }
2773 module_exit(xhci_hcd_cleanup);
2774