xhci.c 149.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * xHCI host controller driver
 *
 * Copyright (C) 2008 Intel Corp.
 *
 * Author: Sarah Sharp
 * Some code borrowed from the Linux EHCI driver.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

D
Dong Nguyen 已提交
23
#include <linux/pci.h>
24
#include <linux/irq.h>
25
#include <linux/log2.h>
26
#include <linux/module.h>
27
#include <linux/moduleparam.h>
28
#include <linux/slab.h>
29
#include <linux/dmi.h>
30
#include <linux/dma-mapping.h>
31 32

#include "xhci.h"
33
#include "xhci-trace.h"
34
#include "xhci-mtk.h"
35 36 37 38

#define DRIVER_AUTHOR "Sarah Sharp"
#define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"

39 40
#define	PORT_WAKE_BITS	(PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)

41 42 43 44 45
/* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
static int link_quirk;
module_param(link_quirk, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");

T
Takashi Iwai 已提交
46 47 48 49
static unsigned int quirks;
module_param(quirks, uint, S_IRUGO);
MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");

50 51
/* TODO: copied from ehci-hcd.c - can this be refactored? */
/*
52
 * xhci_handshake - spin reading hc until handshake completes or fails
53 54 55 56 57 58 59 60 61 62 63
 * @ptr: address of hc register to be read
 * @mask: bits to look at in result of read
 * @done: value of those bits when handshake succeeds
 * @usec: timeout in microseconds
 *
 * Returns negative errno, or zero on success
 *
 * Success happens when the "mask" bits have the specified value (hardware
 * handshake done).  There are two failure modes:  "usec" have passed (major
 * hardware flakeout), or the register reads as all-ones (hardware removed).
 */
64
int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
65 66 67 68
{
	u32	result;

	do {
69
		result = readl(ptr);
70 71 72 73 74 75 76 77 78 79 80 81
		if (result == ~(u32)0)		/* card removed */
			return -ENODEV;
		result &= mask;
		if (result == done)
			return 0;
		udelay(1);
		usec--;
	} while (usec > 0);
	return -ETIMEDOUT;
}

/*
82
 * Disable interrupts and begin the xHCI halting process.
83
 */
84
void xhci_quiesce(struct xhci_hcd *xhci)
85 86 87 88 89 90
{
	u32 halted;
	u32 cmd;
	u32 mask;

	mask = ~(XHCI_IRQS);
91
	halted = readl(&xhci->op_regs->status) & STS_HALT;
92 93 94
	if (!halted)
		mask &= ~CMD_RUN;

95
	cmd = readl(&xhci->op_regs->command);
96
	cmd &= mask;
97
	writel(cmd, &xhci->op_regs->command);
98 99 100 101 102 103 104
}

/*
 * Force HC into halt state.
 *
 * Disable any IRQs and clear the run/stop bit.
 * HC will complete any current and actively pipelined transactions, and
105
 * should halt within 16 ms of the run/stop bit being cleared.
106 107 108 109
 * Read HC Halted bit in the status register to see when the HC is finished.
 */
int xhci_halt(struct xhci_hcd *xhci)
{
110
	int ret;
111
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
112
	xhci_quiesce(xhci);
113

114
	ret = xhci_handshake(&xhci->op_regs->status,
115
			STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
116 117 118 119 120 121
	if (ret) {
		xhci_warn(xhci, "Host halt failed, %d\n", ret);
		return ret;
	}
	xhci->xhc_state |= XHCI_STATE_HALTED;
	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
122
	return ret;
123 124
}

125 126 127
/*
 * Set the run bit and wait for the host to be running.
 */
128
static int xhci_start(struct xhci_hcd *xhci)
129 130 131 132
{
	u32 temp;
	int ret;

133
	temp = readl(&xhci->op_regs->command);
134
	temp |= (CMD_RUN);
135
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
136
			temp);
137
	writel(temp, &xhci->op_regs->command);
138 139 140 141 142

	/*
	 * Wait for the HCHalted Status bit to be 0 to indicate the host is
	 * running.
	 */
143
	ret = xhci_handshake(&xhci->op_regs->status,
144 145 146 147 148
			STS_HALT, 0, XHCI_MAX_HALT_USEC);
	if (ret == -ETIMEDOUT)
		xhci_err(xhci, "Host took too long to start, "
				"waited %u microseconds.\n",
				XHCI_MAX_HALT_USEC);
149
	if (!ret)
150 151
		/* clear state flags. Including dying, halted or removing */
		xhci->xhc_state = 0;
152

153 154 155
	return ret;
}

156
/*
157
 * Reset a halted HC.
158 159 160 161 162 163 164 165 166
 *
 * This resets pipelines, timers, counters, state machines, etc.
 * Transactions will be terminated immediately, and operational registers
 * will be set to their defaults.
 */
int xhci_reset(struct xhci_hcd *xhci)
{
	u32 command;
	u32 state;
167
	int ret, i;
168

169
	state = readl(&xhci->op_regs->status);
170 171 172 173 174 175

	if (state == ~(u32)0) {
		xhci_warn(xhci, "Host not accessible, reset failed.\n");
		return -ENODEV;
	}

176 177 178 179
	if ((state & STS_HALT) == 0) {
		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
		return 0;
	}
180

181
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
182
	command = readl(&xhci->op_regs->command);
183
	command |= CMD_RESET;
184
	writel(command, &xhci->op_regs->command);
185

186 187 188 189 190 191 192 193 194 195
	/* Existing Intel xHCI controllers require a delay of 1 mS,
	 * after setting the CMD_RESET bit, and before accessing any
	 * HC registers. This allows the HC to complete the
	 * reset operation and be ready for HC register access.
	 * Without this delay, the subsequent HC register access,
	 * may result in a system hang very rarely.
	 */
	if (xhci->quirks & XHCI_INTEL_HOST)
		udelay(1000);

196
	ret = xhci_handshake(&xhci->op_regs->command,
197
			CMD_RESET, 0, 10 * 1000 * 1000);
198 199 200
	if (ret)
		return ret;

201 202
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			 "Wait for controller to be ready for doorbell rings");
203 204 205 206
	/*
	 * xHCI cannot write to any doorbells or operational registers other
	 * than status until the "Controller Not Ready" flag is cleared.
	 */
207
	ret = xhci_handshake(&xhci->op_regs->status,
208
			STS_CNR, 0, 10 * 1000 * 1000);
209 210 211 212 213 214 215 216

	for (i = 0; i < 2; ++i) {
		xhci->bus_state[i].port_c_suspend = 0;
		xhci->bus_state[i].suspended_ports = 0;
		xhci->bus_state[i].resuming_ports = 0;
	}

	return ret;
217 218
}

219 220
#ifdef CONFIG_PCI
static int xhci_free_msi(struct xhci_hcd *xhci)
D
Dong Nguyen 已提交
221 222 223
{
	int i;

224 225
	if (!xhci->msix_entries)
		return -EINVAL;
D
Dong Nguyen 已提交
226

227 228 229 230 231
	for (i = 0; i < xhci->msix_count; i++)
		if (xhci->msix_entries[i].vector)
			free_irq(xhci->msix_entries[i].vector,
					xhci_to_hcd(xhci));
	return 0;
D
Dong Nguyen 已提交
232 233 234 235 236 237
}

/*
 * Set up MSI
 */
static int xhci_setup_msi(struct xhci_hcd *xhci)
238 239
{
	int ret;
D
Dong Nguyen 已提交
240 241 242 243
	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);

	ret = pci_enable_msi(pdev);
	if (ret) {
244 245
		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
				"failed to allocate MSI entry");
D
Dong Nguyen 已提交
246 247 248
		return ret;
	}

249
	ret = request_irq(pdev->irq, xhci_msi_irq,
D
Dong Nguyen 已提交
250 251
				0, "xhci_hcd", xhci_to_hcd(xhci));
	if (ret) {
252 253
		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
				"disable MSI interrupt");
D
Dong Nguyen 已提交
254 255 256 257 258 259
		pci_disable_msi(pdev);
	}

	return ret;
}

260 261 262 263 264 265 266 267 268 269
/*
 * Free IRQs
 * free all IRQs request
 */
static void xhci_free_irq(struct xhci_hcd *xhci)
{
	struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
	int ret;

	/* return if using legacy interrupt */
270
	if (xhci_to_hcd(xhci)->irq > 0)
271 272 273 274 275
		return;

	ret = xhci_free_msi(xhci);
	if (!ret)
		return;
276
	if (pdev->irq > 0)
277 278 279 280 281
		free_irq(pdev->irq, xhci_to_hcd(xhci));

	return;
}

D
Dong Nguyen 已提交
282 283 284 285 286 287
/*
 * Set up MSI-X
 */
static int xhci_setup_msix(struct xhci_hcd *xhci)
{
	int i, ret = 0;
288 289
	struct usb_hcd *hcd = xhci_to_hcd(xhci);
	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
290

D
Dong Nguyen 已提交
291 292 293 294 295 296 297 298 299 300 301 302
	/*
	 * calculate number of msi-x vectors supported.
	 * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
	 *   with max number of interrupters based on the xhci HCSPARAMS1.
	 * - num_online_cpus: maximum msi-x vectors per CPUs core.
	 *   Add additional 1 vector to ensure always available interrupt.
	 */
	xhci->msix_count = min(num_online_cpus() + 1,
				HCS_MAX_INTRS(xhci->hcs_params1));

	xhci->msix_entries =
		kmalloc((sizeof(struct msix_entry))*xhci->msix_count,
303
				GFP_KERNEL);
304
	if (!xhci->msix_entries)
305
		return -ENOMEM;
D
Dong Nguyen 已提交
306 307 308 309 310

	for (i = 0; i < xhci->msix_count; i++) {
		xhci->msix_entries[i].entry = i;
		xhci->msix_entries[i].vector = 0;
	}
311

312
	ret = pci_enable_msix_exact(pdev, xhci->msix_entries, xhci->msix_count);
313
	if (ret) {
314 315
		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
				"Failed to enable MSI-X");
316 317 318
		goto free_entries;
	}

D
Dong Nguyen 已提交
319 320
	for (i = 0; i < xhci->msix_count; i++) {
		ret = request_irq(xhci->msix_entries[i].vector,
321
				xhci_msi_irq,
D
Dong Nguyen 已提交
322 323 324
				0, "xhci_hcd", xhci_to_hcd(xhci));
		if (ret)
			goto disable_msix;
325
	}
D
Dong Nguyen 已提交
326

327
	hcd->msix_enabled = 1;
D
Dong Nguyen 已提交
328
	return ret;
329 330

disable_msix:
331
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
D
Dong Nguyen 已提交
332
	xhci_free_irq(xhci);
333 334 335 336 337 338 339 340 341 342
	pci_disable_msix(pdev);
free_entries:
	kfree(xhci->msix_entries);
	xhci->msix_entries = NULL;
	return ret;
}

/* Free any IRQs and disable MSI-X */
static void xhci_cleanup_msix(struct xhci_hcd *xhci)
{
343 344
	struct usb_hcd *hcd = xhci_to_hcd(xhci);
	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
345

346 347 348
	if (xhci->quirks & XHCI_PLAT)
		return;

D
Dong Nguyen 已提交
349 350 351 352 353 354 355 356 357 358
	xhci_free_irq(xhci);

	if (xhci->msix_entries) {
		pci_disable_msix(pdev);
		kfree(xhci->msix_entries);
		xhci->msix_entries = NULL;
	} else {
		pci_disable_msi(pdev);
	}

359
	hcd->msix_enabled = 0;
D
Dong Nguyen 已提交
360
	return;
361 362
}

363
static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)
364 365 366 367 368 369 370 371 372 373 374 375
{
	int i;

	if (xhci->msix_entries) {
		for (i = 0; i < xhci->msix_count; i++)
			synchronize_irq(xhci->msix_entries[i].vector);
	}
}

static int xhci_try_enable_msi(struct usb_hcd *hcd)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
376
	struct pci_dev  *pdev;
377 378
	int ret;

379 380 381 382 383
	/* The xhci platform device has set up IRQs through usb_add_hcd. */
	if (xhci->quirks & XHCI_PLAT)
		return 0;

	pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
384 385 386 387 388
	/*
	 * Some Fresco Logic host controllers advertise MSI, but fail to
	 * generate interrupts.  Don't even try to enable MSI.
	 */
	if (xhci->quirks & XHCI_BROKEN_MSI)
389
		goto legacy_irq;
390 391 392 393

	/* unregister the legacy interrupt */
	if (hcd->irq)
		free_irq(hcd->irq, hcd);
394
	hcd->irq = 0;
395 396 397 398 399 400 401

	ret = xhci_setup_msix(xhci);
	if (ret)
		/* fall back to msi*/
		ret = xhci_setup_msi(xhci);

	if (!ret)
402
		/* hcd->irq is 0, we have MSI */
403 404
		return 0;

405 406 407 408 409
	if (!pdev->irq) {
		xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
		return -EINVAL;
	}

410
 legacy_irq:
411 412 413 414
	if (!strlen(hcd->irq_descr))
		snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
			 hcd->driver->description, hcd->self.busnum);

415 416 417 418 419 420 421 422 423 424 425 426 427 428
	/* fall back to legacy interrupt*/
	ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
			hcd->irq_descr, hcd);
	if (ret) {
		xhci_err(xhci, "request interrupt %d failed\n",
				pdev->irq);
		return ret;
	}
	hcd->irq = pdev->irq;
	return 0;
}

#else

429
static inline int xhci_try_enable_msi(struct usb_hcd *hcd)
430 431 432 433
{
	return 0;
}

434
static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)
435 436 437
{
}

438
static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
439 440 441 442 443
{
}

#endif

444 445 446 447 448 449 450 451 452 453
static void compliance_mode_recovery(unsigned long arg)
{
	struct xhci_hcd *xhci;
	struct usb_hcd *hcd;
	u32 temp;
	int i;

	xhci = (struct xhci_hcd *)arg;

	for (i = 0; i < xhci->num_usb3_ports; i++) {
454
		temp = readl(xhci->usb3_ports[i]);
455 456 457 458 459
		if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
			/*
			 * Compliance Mode Detected. Letting USB Core
			 * handle the Warm Reset
			 */
460 461
			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
					"Compliance mode detected->port %d",
462
					i + 1);
463 464
			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
					"Attempting compliance mode recovery");
465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
			hcd = xhci->shared_hcd;

			if (hcd->state == HC_STATE_SUSPENDED)
				usb_hcd_resume_root_hub(hcd);

			usb_hcd_poll_rh_status(hcd);
		}
	}

	if (xhci->port_status_u0 != ((1 << xhci->num_usb3_ports)-1))
		mod_timer(&xhci->comp_mode_recovery_timer,
			jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
}

/*
 * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
 * that causes ports behind that hardware to enter compliance mode sometimes.
 * The quirk creates a timer that polls every 2 seconds the link state of
 * each host controller's port and recovers it by issuing a Warm reset
 * if Compliance mode is detected, otherwise the port will become "dead" (no
 * device connections or disconnections will be detected anymore). Becasue no
 * status event is generated when entering compliance mode (per xhci spec),
 * this quirk is needed on systems that have the failing hardware installed.
 */
static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
{
	xhci->port_status_u0 = 0;
J
Julia Lawall 已提交
492 493
	setup_timer(&xhci->comp_mode_recovery_timer,
		    compliance_mode_recovery, (unsigned long)xhci);
494 495 496 497
	xhci->comp_mode_recovery_timer.expires = jiffies +
			msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);

	add_timer(&xhci->comp_mode_recovery_timer);
498 499
	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
			"Compliance mode recovery timer initialized");
500 501 502 503 504 505 506 507
}

/*
 * This function identifies the systems that have installed the SN65LVPE502CP
 * USB3.0 re-driver and that need the Compliance Mode Quirk.
 * Systems:
 * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
 */
508
static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
509 510 511 512 513
{
	const char *dmi_product_name, *dmi_sys_vendor;

	dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
	dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
514 515
	if (!dmi_product_name || !dmi_sys_vendor)
		return false;
516 517 518 519 520 521

	if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
		return false;

	if (strstr(dmi_product_name, "Z420") ||
			strstr(dmi_product_name, "Z620") ||
522
			strstr(dmi_product_name, "Z820") ||
523
			strstr(dmi_product_name, "Z1 Workstation"))
524 525 526 527 528 529 530 531 532 533 534
		return true;

	return false;
}

static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
{
	return (xhci->port_status_u0 == ((1 << xhci->num_usb3_ports)-1));
}


535 536 537 538 539 540 541 542 543 544 545 546
/*
 * Initialize memory for HCD and xHC (one-time init).
 *
 * Program the PAGESIZE register, initialize the device context array, create
 * device contexts (?), set up a command ring segment (or two?), create event
 * ring (one for now).
 */
int xhci_init(struct usb_hcd *hcd)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
	int retval = 0;

547
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
548
	spin_lock_init(&xhci->lock);
549
	if (xhci->hci_version == 0x95 && link_quirk) {
550 551
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"QUIRK: Not clearing Link TRB chain bits.");
552 553
		xhci->quirks |= XHCI_LINK_TRB_QUIRK;
	} else {
554 555
		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
				"xHCI doesn't need link TRB QUIRK");
556
	}
557
	retval = xhci_mem_init(xhci, GFP_KERNEL);
558
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
559

560
	/* Initializing Compliance Mode Recovery Data If Needed */
561
	if (xhci_compliance_mode_recovery_timer_quirk_check()) {
562 563 564 565
		xhci->quirks |= XHCI_COMP_MODE_QUIRK;
		compliance_mode_recovery_timer_init(xhci);
	}

566 567 568
	return retval;
}

569 570 571
/*-------------------------------------------------------------------------*/


572 573 574 575 576 577 578
static int xhci_run_finished(struct xhci_hcd *xhci)
{
	if (xhci_start(xhci)) {
		xhci_halt(xhci);
		return -ENODEV;
	}
	xhci->shared_hcd->state = HC_STATE_RUNNING;
E
Elric Fu 已提交
579
	xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
580 581 582 583

	if (xhci->quirks & XHCI_NEC_HOST)
		xhci_ring_cmd_db(xhci);

584 585
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"Finished xhci_run for USB3 roothub");
586 587 588
	return 0;
}

589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
/*
 * Start the HC after it was halted.
 *
 * This function is called by the USB core when the HC driver is added.
 * Its opposite is xhci_stop().
 *
 * xhci_init() must be called once before this function can be called.
 * Reset the HC, enable device slot contexts, program DCBAAP, and
 * set command ring pointer and event ring pointer.
 *
 * Setup MSI-X vectors and enable interrupts.
 */
int xhci_run(struct usb_hcd *hcd)
{
	u32 temp;
604
	u64 temp_64;
605
	int ret;
606 607
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);

608 609 610
	/* Start the xHCI host controller running only after the USB 2.0 roothub
	 * is setup.
	 */
611

S
Sarah Sharp 已提交
612
	hcd->uses_new_polling = 1;
613 614
	if (!usb_hcd_is_primary_hcd(hcd))
		return xhci_run_finished(xhci);
S
Sarah Sharp 已提交
615

616
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
D
Dong Nguyen 已提交
617

618
	ret = xhci_try_enable_msi(hcd);
D
Dong Nguyen 已提交
619
	if (ret)
620
		return ret;
621

622 623 624 625 626 627 628 629 630 631
	xhci_dbg(xhci, "Command ring memory map follows:\n");
	xhci_debug_ring(xhci, xhci->cmd_ring);
	xhci_dbg_ring_ptrs(xhci, xhci->cmd_ring);
	xhci_dbg_cmd_ptrs(xhci);

	xhci_dbg(xhci, "ERST memory map follows:\n");
	xhci_dbg_erst(xhci, &xhci->erst);
	xhci_dbg(xhci, "Event ring:\n");
	xhci_debug_ring(xhci, xhci->event_ring);
	xhci_dbg_ring_ptrs(xhci, xhci->event_ring);
632
	temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
633
	temp_64 &= ~ERST_PTR_MASK;
634 635
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"ERST deq = 64'h%0lx", (long unsigned int) temp_64);
636

637 638
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"// Set the interrupt modulation register");
639
	temp = readl(&xhci->ir_set->irq_control);
640
	temp &= ~ER_IRQ_INTERVAL_MASK;
641 642 643 644 645
	/*
	 * the increment interval is 8 times as much as that defined
	 * in xHCI spec on MTK's controller
	 */
	temp |= (u32) ((xhci->quirks & XHCI_MTK_HOST) ? 20 : 160);
646
	writel(temp, &xhci->ir_set->irq_control);
647 648

	/* Set the HCD state before we enable the irqs */
649
	temp = readl(&xhci->op_regs->command);
650
	temp |= (CMD_EIE);
651 652
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"// Enable interrupts, cmd = 0x%x.", temp);
653
	writel(temp, &xhci->op_regs->command);
654

655
	temp = readl(&xhci->ir_set->irq_pending);
656 657
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"// Enabling event ring interrupter %p by writing 0x%x to irq_pending",
658
			xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
659
	writel(ER_IRQ_ENABLE(temp), &xhci->ir_set->irq_pending);
660
	xhci_print_ir_set(xhci, 0);
661

662 663 664 665 666 667
	if (xhci->quirks & XHCI_NEC_HOST) {
		struct xhci_command *command;
		command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
		if (!command)
			return -ENOMEM;
		xhci_queue_vendor_command(xhci, command, 0, 0, 0,
668
				TRB_TYPE(TRB_NEC_GET_FW));
669
	}
670 671
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"Finished xhci_run for USB2 roothub");
672 673
	return 0;
}
674
EXPORT_SYMBOL_GPL(xhci_run);
675

676 677 678 679 680 681 682 683 684 685 686 687 688 689
/*
 * Stop xHCI driver.
 *
 * This function is called by the USB core when the HC driver is removed.
 * Its opposite is xhci_run().
 *
 * Disable device contexts, disable IRQs, and quiesce the HC.
 * Reset the HC, finish any completed transactions, and cleanup memory.
 */
void xhci_stop(struct usb_hcd *hcd)
{
	u32 temp;
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);

690 691
	mutex_lock(&xhci->mutex);

692 693 694 695 696 697 698 699 700 701 702 703 704 705
	if (!(xhci->xhc_state & XHCI_STATE_HALTED)) {
		spin_lock_irq(&xhci->lock);

		xhci->xhc_state |= XHCI_STATE_HALTED;
		xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
		xhci_halt(xhci);
		xhci_reset(xhci);
		spin_unlock_irq(&xhci->lock);
	}

	if (!usb_hcd_is_primary_hcd(hcd)) {
		mutex_unlock(&xhci->mutex);
		return;
	}
706

707 708
	xhci_cleanup_msix(xhci);

709 710
	/* Deleting Compliance Mode Recovery Timer */
	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
711
			(!(xhci_all_ports_seen_u0(xhci)))) {
712
		del_timer_sync(&xhci->comp_mode_recovery_timer);
713 714
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"%s: compliance mode recovery timer deleted",
715 716
				__func__);
	}
717

A
Andiry Xu 已提交
718 719 720
	if (xhci->quirks & XHCI_AMD_PLL_FIX)
		usb_amd_dev_put();

721 722
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"// Disabling event ring interrupts");
723
	temp = readl(&xhci->op_regs->status);
724
	writel(temp & ~STS_EINT, &xhci->op_regs->status);
725
	temp = readl(&xhci->ir_set->irq_pending);
726
	writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
727
	xhci_print_ir_set(xhci, 0);
728

729
	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
730
	xhci_mem_cleanup(xhci);
731 732
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"xhci_stop completed - status = %x",
733
			readl(&xhci->op_regs->status));
734
	mutex_unlock(&xhci->mutex);
735 736 737 738 739 740 741 742
}

/*
 * Shutdown HC (not bus-specific)
 *
 * This is called when the machine is rebooting or halting.  We assume that the
 * machine will be powered off, and the HC's internal state will be reset.
 * Don't bother to free memory.
743 744
 *
 * This will only ever be called with the main usb_hcd (the USB3 roothub).
745 746 747 748 749
 */
void xhci_shutdown(struct usb_hcd *hcd)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);

750
	if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
751 752
		usb_disable_xhci_ports(to_pci_dev(hcd->self.controller));

753 754
	spin_lock_irq(&xhci->lock);
	xhci_halt(xhci);
755 756 757
	/* Workaround for spurious wakeups at shutdown with HSW */
	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
		xhci_reset(xhci);
D
Dong Nguyen 已提交
758
	spin_unlock_irq(&xhci->lock);
759

760 761
	xhci_cleanup_msix(xhci);

762 763
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"xhci_shutdown completed - status = %x",
764
			readl(&xhci->op_regs->status));
765 766 767 768

	/* Yet another workaround for spurious wakeups at shutdown with HSW */
	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
		pci_set_power_state(to_pci_dev(hcd->self.controller), PCI_D3hot);
769 770
}

771
#ifdef CONFIG_PM
772 773
static void xhci_save_registers(struct xhci_hcd *xhci)
{
774 775
	xhci->s3.command = readl(&xhci->op_regs->command);
	xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
776
	xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
777 778
	xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
	xhci->s3.erst_size = readl(&xhci->ir_set->erst_size);
779 780
	xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
	xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
781 782
	xhci->s3.irq_pending = readl(&xhci->ir_set->irq_pending);
	xhci->s3.irq_control = readl(&xhci->ir_set->irq_control);
783 784 785 786
}

static void xhci_restore_registers(struct xhci_hcd *xhci)
{
787 788
	writel(xhci->s3.command, &xhci->op_regs->command);
	writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
789
	xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
790 791
	writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
	writel(xhci->s3.erst_size, &xhci->ir_set->erst_size);
792 793
	xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
	xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);
794 795
	writel(xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
	writel(xhci->s3.irq_control, &xhci->ir_set->irq_control);
796 797
}

798 799 800 801 802
static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
{
	u64	val_64;

	/* step 2: initialize command ring buffer */
803
	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
804 805 806 807 808
	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
		(xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
				      xhci->cmd_ring->dequeue) &
		 (u64) ~CMD_RING_RSVD_BITS) |
		xhci->cmd_ring->cycle_state;
809 810
	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
			"// Setting command ring address to 0x%llx",
811
			(long unsigned long) val_64);
812
	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
}

/*
 * The whole command ring must be cleared to zero when we suspend the host.
 *
 * The host doesn't save the command ring pointer in the suspend well, so we
 * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
 * aligned, because of the reserved bits in the command ring dequeue pointer
 * register.  Therefore, we can't just set the dequeue pointer back in the
 * middle of the ring (TRBs are 16-byte aligned).
 */
static void xhci_clear_command_ring(struct xhci_hcd *xhci)
{
	struct xhci_ring *ring;
	struct xhci_segment *seg;

	ring = xhci->cmd_ring;
	seg = ring->deq_seg;
	do {
832 833 834 835
		memset(seg->trbs, 0,
			sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
		seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
			cpu_to_le32(~TRB_CYCLE);
836 837 838 839 840 841 842 843 844
		seg = seg->next;
	} while (seg != ring->deq_seg);

	/* Reset the software enqueue and dequeue pointers */
	ring->deq_seg = ring->first_seg;
	ring->dequeue = ring->first_seg->trbs;
	ring->enq_seg = ring->deq_seg;
	ring->enqueue = ring->dequeue;

845
	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
	/*
	 * Ring is now zeroed, so the HW should look for change of ownership
	 * when the cycle bit is set to 1.
	 */
	ring->cycle_state = 1;

	/*
	 * Reset the hardware dequeue pointer.
	 * Yes, this will need to be re-written after resume, but we're paranoid
	 * and want to make sure the hardware doesn't access bogus memory
	 * because, say, the BIOS or an SMI started the host without changing
	 * the command ring pointers.
	 */
	xhci_set_cmd_ring_deq(xhci);
}

862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
static void xhci_disable_port_wake_on_bits(struct xhci_hcd *xhci)
{
	int port_index;
	__le32 __iomem **port_array;
	unsigned long flags;
	u32 t1, t2;

	spin_lock_irqsave(&xhci->lock, flags);

	/* disble usb3 ports Wake bits*/
	port_index = xhci->num_usb3_ports;
	port_array = xhci->usb3_ports;
	while (port_index--) {
		t1 = readl(port_array[port_index]);
		t1 = xhci_port_state_to_neutral(t1);
		t2 = t1 & ~PORT_WAKE_BITS;
		if (t1 != t2)
			writel(t2, port_array[port_index]);
	}

	/* disble usb2 ports Wake bits*/
	port_index = xhci->num_usb2_ports;
	port_array = xhci->usb2_ports;
	while (port_index--) {
		t1 = readl(port_array[port_index]);
		t1 = xhci_port_state_to_neutral(t1);
		t2 = t1 & ~PORT_WAKE_BITS;
		if (t1 != t2)
			writel(t2, port_array[port_index]);
	}

	spin_unlock_irqrestore(&xhci->lock, flags);
}

896 897 898 899 900 901
/*
 * Stop HC (not bus-specific)
 *
 * This is called when the machine transition into S3/S4 mode.
 *
 */
902
int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
903 904
{
	int			rc = 0;
905
	unsigned int		delay = XHCI_MAX_HALT_USEC;
906 907 908
	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
	u32			command;

909 910 911
	if (!hcd->state)
		return 0;

912 913 914 915
	if (hcd->state != HC_STATE_SUSPENDED ||
			xhci->shared_hcd->state != HC_STATE_SUSPENDED)
		return -EINVAL;

916 917 918 919
	/* Clear root port wake on bits if wakeup not allowed. */
	if (!do_wakeup)
		xhci_disable_port_wake_on_bits(xhci);

920 921 922 923
	/* Don't poll the roothubs on bus suspend. */
	xhci_dbg(xhci, "%s: stopping port polling.\n", __func__);
	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
	del_timer_sync(&hcd->rh_timer);
924 925
	clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
	del_timer_sync(&xhci->shared_hcd->rh_timer);
926

927 928
	spin_lock_irq(&xhci->lock);
	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
929
	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
930 931 932 933
	/* step 1: stop endpoint */
	/* skipped assuming that port suspend has done */

	/* step 2: clear Run/Stop bit */
934
	command = readl(&xhci->op_regs->command);
935
	command &= ~CMD_RUN;
936
	writel(command, &xhci->op_regs->command);
937 938 939 940

	/* Some chips from Fresco Logic need an extraordinary delay */
	delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;

941
	if (xhci_handshake(&xhci->op_regs->status,
942
		      STS_HALT, STS_HALT, delay)) {
943 944 945 946
		xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
		spin_unlock_irq(&xhci->lock);
		return -ETIMEDOUT;
	}
947
	xhci_clear_command_ring(xhci);
948 949 950 951 952

	/* step 3: save registers */
	xhci_save_registers(xhci);

	/* step 4: set CSS flag */
953
	command = readl(&xhci->op_regs->command);
954
	command |= CMD_CSS;
955
	writel(command, &xhci->op_regs->command);
956
	if (xhci_handshake(&xhci->op_regs->status,
957
				STS_SAVE, 0, 10 * 1000)) {
958
		xhci_warn(xhci, "WARN: xHC save state timeout\n");
959 960 961 962 963
		spin_unlock_irq(&xhci->lock);
		return -ETIMEDOUT;
	}
	spin_unlock_irq(&xhci->lock);

964 965 966 967 968 969 970
	/*
	 * Deleting Compliance Mode Recovery Timer because the xHCI Host
	 * is about to be suspended.
	 */
	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
			(!(xhci_all_ports_seen_u0(xhci)))) {
		del_timer_sync(&xhci->comp_mode_recovery_timer);
971 972
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"%s: compliance mode recovery timer deleted",
973
				__func__);
974 975
	}

976 977
	/* step 5: remove core well power */
	/* synchronize irq when using MSI-X */
978
	xhci_msix_sync_irqs(xhci);
979

980 981
	return rc;
}
982
EXPORT_SYMBOL_GPL(xhci_suspend);
983 984 985 986 987 988 989 990 991

/*
 * start xHC (not bus-specific)
 *
 * This is called when the machine transition from S3/S4 mode.
 *
 */
int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
{
992
	u32			command, temp = 0, status;
993
	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
994
	struct usb_hcd		*secondary_hcd;
995
	int			retval = 0;
996
	bool			comp_timer_running = false;
997

998 999 1000
	if (!hcd->state)
		return 0;

1001
	/* Wait a bit if either of the roothubs need to settle from the
L
Lucas De Marchi 已提交
1002
	 * transition into bus suspend.
1003
	 */
1004 1005 1006
	if (time_before(jiffies, xhci->bus_state[0].next_statechange) ||
			time_before(jiffies,
				xhci->bus_state[1].next_statechange))
1007 1008
		msleep(100);

1009 1010 1011
	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
	set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);

1012
	spin_lock_irq(&xhci->lock);
1013 1014
	if (xhci->quirks & XHCI_RESET_ON_RESUME)
		hibernated = true;
1015 1016 1017 1018 1019

	if (!hibernated) {
		/* step 1: restore register */
		xhci_restore_registers(xhci);
		/* step 2: initialize command ring buffer */
1020
		xhci_set_cmd_ring_deq(xhci);
1021 1022
		/* step 3: restore state and start state*/
		/* step 3: set CRS flag */
1023
		command = readl(&xhci->op_regs->command);
1024
		command |= CMD_CRS;
1025
		writel(command, &xhci->op_regs->command);
1026
		if (xhci_handshake(&xhci->op_regs->status,
1027 1028
			      STS_RESTORE, 0, 10 * 1000)) {
			xhci_warn(xhci, "WARN: xHC restore state timeout\n");
1029 1030 1031
			spin_unlock_irq(&xhci->lock);
			return -ETIMEDOUT;
		}
1032
		temp = readl(&xhci->op_regs->status);
1033 1034 1035 1036
	}

	/* If restore operation fails, re-initialize the HC during resume */
	if ((temp & STS_SRE) || hibernated) {
1037 1038 1039 1040

		if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
				!(xhci_all_ports_seen_u0(xhci))) {
			del_timer_sync(&xhci->comp_mode_recovery_timer);
1041 1042
			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Compliance Mode Recovery Timer deleted!");
1043 1044
		}

1045 1046 1047
		/* Let the USB core know _both_ roothubs lost power. */
		usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
		usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
1048 1049 1050 1051 1052

		xhci_dbg(xhci, "Stop HCD\n");
		xhci_halt(xhci);
		xhci_reset(xhci);
		spin_unlock_irq(&xhci->lock);
1053
		xhci_cleanup_msix(xhci);
1054 1055

		xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1056
		temp = readl(&xhci->op_regs->status);
1057
		writel(temp & ~STS_EINT, &xhci->op_regs->status);
1058
		temp = readl(&xhci->ir_set->irq_pending);
1059
		writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
1060
		xhci_print_ir_set(xhci, 0);
1061 1062 1063 1064

		xhci_dbg(xhci, "cleaning up memory\n");
		xhci_mem_cleanup(xhci);
		xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1065
			    readl(&xhci->op_regs->status));
1066

1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
		/* USB core calls the PCI reinit and start functions twice:
		 * first with the primary HCD, and then with the secondary HCD.
		 * If we don't do the same, the host will never be started.
		 */
		if (!usb_hcd_is_primary_hcd(hcd))
			secondary_hcd = hcd;
		else
			secondary_hcd = xhci->shared_hcd;

		xhci_dbg(xhci, "Initialize the xhci_hcd\n");
		retval = xhci_init(hcd->primary_hcd);
1078 1079
		if (retval)
			return retval;
1080 1081
		comp_timer_running = true;

1082 1083
		xhci_dbg(xhci, "Start the primary HCD\n");
		retval = xhci_run(hcd->primary_hcd);
1084
		if (!retval) {
1085 1086
			xhci_dbg(xhci, "Start the secondary HCD\n");
			retval = xhci_run(secondary_hcd);
1087
		}
1088
		hcd->state = HC_STATE_SUSPENDED;
1089
		xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1090
		goto done;
1091 1092 1093
	}

	/* step 4: set Run/Stop bit */
1094
	command = readl(&xhci->op_regs->command);
1095
	command |= CMD_RUN;
1096
	writel(command, &xhci->op_regs->command);
1097
	xhci_handshake(&xhci->op_regs->status, STS_HALT,
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
		  0, 250 * 1000);

	/* step 5: walk topology and initialize portsc,
	 * portpmsc and portli
	 */
	/* this is done in bus_resume */

	/* step 6: restart each of the previously
	 * Running endpoints by ringing their doorbells
	 */

	spin_unlock_irq(&xhci->lock);
1110 1111 1112

 done:
	if (retval == 0) {
1113 1114 1115 1116
		/* Resume root hubs only when have pending events. */
		status = readl(&xhci->op_regs->status);
		if (status & STS_EINT) {
			usb_hcd_resume_root_hub(xhci->shared_hcd);
M
Mathias Nyman 已提交
1117
			usb_hcd_resume_root_hub(hcd);
1118
		}
1119
	}
1120 1121 1122 1123 1124 1125 1126

	/*
	 * If system is subject to the Quirk, Compliance Mode Timer needs to
	 * be re-initialized Always after a system resume. Ports are subject
	 * to suffer the Compliance Mode issue again. It doesn't matter if
	 * ports have entered previously to U0 before system's suspension.
	 */
1127
	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1128 1129
		compliance_mode_recovery_timer_init(xhci);

1130 1131
	/* Re-enable port polling. */
	xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1132 1133
	set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
	usb_hcd_poll_rh_status(xhci->shared_hcd);
M
Mathias Nyman 已提交
1134 1135
	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
	usb_hcd_poll_rh_status(hcd);
1136

1137
	return retval;
1138
}
1139
EXPORT_SYMBOL_GPL(xhci_resume);
1140 1141
#endif	/* CONFIG_PM */

1142 1143
/*-------------------------------------------------------------------------*/

1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
/**
 * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
 * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
 * value to right shift 1 for the bitmask.
 *
 * Index  = (epnum * 2) + direction - 1,
 * where direction = 0 for OUT, 1 for IN.
 * For control endpoints, the IN index is used (OUT index is unused), so
 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
 */
unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
{
	unsigned int index;
	if (usb_endpoint_xfer_control(desc))
		index = (unsigned int) (usb_endpoint_num(desc)*2);
	else
		index = (unsigned int) (usb_endpoint_num(desc)*2) +
			(usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
	return index;
}

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
/* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint
 * address from the XHCI endpoint index.
 */
unsigned int xhci_get_endpoint_address(unsigned int ep_index)
{
	unsigned int number = DIV_ROUND_UP(ep_index, 2);
	unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
	return direction | number;
}

1175 1176 1177 1178 1179 1180 1181 1182 1183
/* Find the flag for this endpoint (for use in the control context).  Use the
 * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
 * bit 1, etc.
 */
unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
{
	return 1 << (xhci_get_endpoint_index(desc) + 1);
}

1184 1185 1186 1187 1188 1189 1190 1191 1192
/* Find the flag for this endpoint (for use in the control context).  Use the
 * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
 * bit 1, etc.
 */
unsigned int xhci_get_endpoint_flag_from_index(unsigned int ep_index)
{
	return 1 << (ep_index + 1);
}

1193 1194 1195 1196 1197 1198
/* Compute the last valid endpoint context index.  Basically, this is the
 * endpoint index plus one.  For slot contexts with more than valid endpoint,
 * we find the most significant bit set in the added contexts flags.
 * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
 * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
 */
1199
unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1200 1201 1202 1203
{
	return fls(added_ctxs) - 1;
}

1204 1205 1206
/* Returns 1 if the arguments are OK;
 * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
 */
1207
static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1208 1209 1210 1211 1212
		struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
		const char *func) {
	struct xhci_hcd	*xhci;
	struct xhci_virt_device	*virt_dev;

1213
	if (!hcd || (check_ep && !ep) || !udev) {
1214
		pr_debug("xHCI %s called with invalid args\n", func);
1215 1216 1217
		return -EINVAL;
	}
	if (!udev->parent) {
1218
		pr_debug("xHCI %s called for root hub\n", func);
1219 1220
		return 0;
	}
1221

1222
	xhci = hcd_to_xhci(hcd);
1223
	if (check_virt_dev) {
1224
		if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1225 1226
			xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
					func);
1227 1228 1229 1230 1231
			return -EINVAL;
		}

		virt_dev = xhci->devs[udev->slot_id];
		if (virt_dev->udev != udev) {
1232
			xhci_dbg(xhci, "xHCI %s called with udev and "
1233 1234 1235
					  "virt_dev does not match\n", func);
			return -EINVAL;
		}
1236
	}
1237

1238 1239 1240
	if (xhci->xhc_state & XHCI_STATE_HALTED)
		return -ENODEV;

1241 1242 1243
	return 1;
}

1244
static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1245 1246
		struct usb_device *udev, struct xhci_command *command,
		bool ctx_change, bool must_succeed);
1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259

/*
 * Full speed devices may have a max packet size greater than 8 bytes, but the
 * USB core doesn't know that until it reads the first 8 bytes of the
 * descriptor.  If the usb_device's max packet size changes after that point,
 * we need to issue an evaluate context command and wait on it.
 */
static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
		unsigned int ep_index, struct urb *urb)
{
	struct xhci_container_ctx *out_ctx;
	struct xhci_input_control_ctx *ctrl_ctx;
	struct xhci_ep_ctx *ep_ctx;
1260
	struct xhci_command *command;
1261 1262 1263 1264 1265 1266
	int max_packet_size;
	int hw_max_packet_size;
	int ret = 0;

	out_ctx = xhci->devs[slot_id]->out_ctx;
	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
M
Matt Evans 已提交
1267
	hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1268
	max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
1269
	if (hw_max_packet_size != max_packet_size) {
1270 1271 1272 1273
		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
				"Max Packet Size for ep 0 changed.");
		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
				"Max packet size in usb_device = %d",
1274
				max_packet_size);
1275 1276
		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
				"Max packet size in xHCI HW = %d",
1277
				hw_max_packet_size);
1278 1279
		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
				"Issuing evaluate context command.");
1280

1281 1282 1283 1284
		/* Set up the input context flags for the command */
		/* FIXME: This won't work if a non-default control endpoint
		 * changes max packet sizes.
		 */
1285 1286 1287 1288 1289 1290

		command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
		if (!command)
			return -ENOMEM;

		command->in_ctx = xhci->devs[slot_id]->in_ctx;
1291
		ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
1292 1293 1294
		if (!ctrl_ctx) {
			xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
					__func__);
1295 1296
			ret = -ENOMEM;
			goto command_cleanup;
1297
		}
1298
		/* Set up the modified control endpoint 0 */
1299 1300
		xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
				xhci->devs[slot_id]->out_ctx, ep_index);
1301

1302
		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
M
Matt Evans 已提交
1303 1304
		ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
		ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1305

M
Matt Evans 已提交
1306
		ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1307 1308 1309
		ctrl_ctx->drop_flags = 0;

		xhci_dbg(xhci, "Slot %d input context\n", slot_id);
1310
		xhci_dbg_ctx(xhci, command->in_ctx, ep_index);
1311 1312 1313
		xhci_dbg(xhci, "Slot %d output context\n", slot_id);
		xhci_dbg_ctx(xhci, out_ctx, ep_index);

1314
		ret = xhci_configure_endpoint(xhci, urb->dev, command,
1315
				true, false);
1316 1317 1318 1319

		/* Clean up the input context for later use by bandwidth
		 * functions.
		 */
M
Matt Evans 已提交
1320
		ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1321 1322 1323
command_cleanup:
		kfree(command->completion);
		kfree(command);
1324 1325 1326 1327
	}
	return ret;
}

1328 1329 1330 1331 1332 1333 1334
/*
 * non-error returns are a promise to giveback() the urb later
 * we drop ownership so next owner (or urb unlink) can get it
 */
int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
A
Andiry Xu 已提交
1335
	struct xhci_td *buffer;
1336 1337 1338
	unsigned long flags;
	int ret = 0;
	unsigned int slot_id, ep_index;
1339 1340
	struct urb_priv	*urb_priv;
	int size, i;
1341

1342 1343
	if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
					true, true, __func__) <= 0)
1344 1345 1346 1347 1348
		return -EINVAL;

	slot_id = urb->dev->slot_id;
	ep_index = xhci_get_endpoint_index(&urb->ep->desc);

1349
	if (!HCD_HW_ACCESSIBLE(hcd)) {
1350 1351 1352 1353 1354
		if (!in_interrupt())
			xhci_dbg(xhci, "urb submitted during PCI suspend\n");
		ret = -ESHUTDOWN;
		goto exit;
	}
1355 1356 1357

	if (usb_endpoint_xfer_isoc(&urb->ep->desc))
		size = urb->number_of_packets;
1358 1359 1360 1361 1362
	else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&
	    urb->transfer_buffer_length > 0 &&
	    urb->transfer_flags & URB_ZERO_PACKET &&
	    !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))
		size = 2;
1363 1364 1365 1366 1367 1368 1369 1370
	else
		size = 1;

	urb_priv = kzalloc(sizeof(struct urb_priv) +
				  size * sizeof(struct xhci_td *), mem_flags);
	if (!urb_priv)
		return -ENOMEM;

A
Andiry Xu 已提交
1371 1372 1373 1374 1375 1376
	buffer = kzalloc(size * sizeof(struct xhci_td), mem_flags);
	if (!buffer) {
		kfree(urb_priv);
		return -ENOMEM;
	}

1377
	for (i = 0; i < size; i++) {
A
Andiry Xu 已提交
1378 1379
		urb_priv->td[i] = buffer;
		buffer++;
1380 1381 1382 1383 1384 1385
	}

	urb_priv->length = size;
	urb_priv->td_cnt = 0;
	urb->hcpriv = urb_priv;

1386 1387 1388 1389 1390 1391 1392
	if (usb_endpoint_xfer_control(&urb->ep->desc)) {
		/* Check to see if the max packet size for the default control
		 * endpoint changed during FS device enumeration
		 */
		if (urb->dev->speed == USB_SPEED_FULL) {
			ret = xhci_check_maxpacket(xhci, slot_id,
					ep_index, urb);
1393
			if (ret < 0) {
1394
				xhci_urb_free_priv(urb_priv);
1395
				urb->hcpriv = NULL;
1396
				return ret;
1397
			}
1398 1399
		}

1400 1401 1402
		/* We have a spinlock and interrupts disabled, so we must pass
		 * atomic context to this function, which may allocate memory.
		 */
1403
		spin_lock_irqsave(&xhci->lock, flags);
1404 1405
		if (xhci->xhc_state & XHCI_STATE_DYING)
			goto dying;
1406
		ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1407
				slot_id, ep_index);
1408 1409
		if (ret)
			goto free_priv;
1410 1411 1412
		spin_unlock_irqrestore(&xhci->lock, flags);
	} else if (usb_endpoint_xfer_bulk(&urb->ep->desc)) {
		spin_lock_irqsave(&xhci->lock, flags);
1413 1414
		if (xhci->xhc_state & XHCI_STATE_DYING)
			goto dying;
1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
		if (xhci->devs[slot_id]->eps[ep_index].ep_state &
				EP_GETTING_STREAMS) {
			xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
					"is transitioning to using streams.\n");
			ret = -EINVAL;
		} else if (xhci->devs[slot_id]->eps[ep_index].ep_state &
				EP_GETTING_NO_STREAMS) {
			xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
					"is transitioning to "
					"not having streams.\n");
			ret = -EINVAL;
		} else {
			ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
					slot_id, ep_index);
		}
1430 1431
		if (ret)
			goto free_priv;
1432
		spin_unlock_irqrestore(&xhci->lock, flags);
1433 1434
	} else if (usb_endpoint_xfer_int(&urb->ep->desc)) {
		spin_lock_irqsave(&xhci->lock, flags);
1435 1436
		if (xhci->xhc_state & XHCI_STATE_DYING)
			goto dying;
1437 1438
		ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
				slot_id, ep_index);
1439 1440
		if (ret)
			goto free_priv;
1441
		spin_unlock_irqrestore(&xhci->lock, flags);
1442
	} else {
A
Andiry Xu 已提交
1443 1444 1445 1446 1447
		spin_lock_irqsave(&xhci->lock, flags);
		if (xhci->xhc_state & XHCI_STATE_DYING)
			goto dying;
		ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
				slot_id, ep_index);
1448 1449
		if (ret)
			goto free_priv;
A
Andiry Xu 已提交
1450
		spin_unlock_irqrestore(&xhci->lock, flags);
1451
	}
1452 1453
exit:
	return ret;
1454 1455 1456 1457
dying:
	xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for "
			"non-responsive xHCI host.\n",
			urb->ep->desc.bEndpointAddress, urb);
1458 1459
	ret = -ESHUTDOWN;
free_priv:
1460
	xhci_urb_free_priv(urb_priv);
1461
	urb->hcpriv = NULL;
1462
	spin_unlock_irqrestore(&xhci->lock, flags);
1463
	return ret;
1464 1465
}

1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495
/*
 * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
 * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
 * should pick up where it left off in the TD, unless a Set Transfer Ring
 * Dequeue Pointer is issued.
 *
 * The TRBs that make up the buffers for the canceled URB will be "removed" from
 * the ring.  Since the ring is a contiguous structure, they can't be physically
 * removed.  Instead, there are two options:
 *
 *  1) If the HC is in the middle of processing the URB to be canceled, we
 *     simply move the ring's dequeue pointer past those TRBs using the Set
 *     Transfer Ring Dequeue Pointer command.  This will be the common case,
 *     when drivers timeout on the last submitted URB and attempt to cancel.
 *
 *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
 *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
 *     HC will need to invalidate the any TRBs it has cached after the stop
 *     endpoint command, as noted in the xHCI 0.95 errata.
 *
 *  3) The TD may have completed by the time the Stop Endpoint Command
 *     completes, so software needs to handle that case too.
 *
 * This function should protect against the TD enqueueing code ringing the
 * doorbell while this code is waiting for a Stop Endpoint command to complete.
 * It also needs to account for multiple cancellations on happening at the same
 * time for the same endpoint.
 *
 * Note that this function can be called in any context, or so says
 * usb_hcd_unlink_urb()
1496 1497 1498
 */
int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
{
1499
	unsigned long flags;
1500
	int ret, i;
1501
	u32 temp;
1502
	struct xhci_hcd *xhci;
1503
	struct urb_priv	*urb_priv;
1504 1505 1506
	struct xhci_td *td;
	unsigned int ep_index;
	struct xhci_ring *ep_ring;
1507
	struct xhci_virt_ep *ep;
1508
	struct xhci_command *command;
1509 1510 1511 1512 1513 1514 1515

	xhci = hcd_to_xhci(hcd);
	spin_lock_irqsave(&xhci->lock, flags);
	/* Make sure the URB hasn't completed or been unlinked already */
	ret = usb_hcd_check_unlink_urb(hcd, urb, status);
	if (ret || !urb->hcpriv)
		goto done;
1516
	temp = readl(&xhci->op_regs->status);
1517
	if (temp == 0xffffffff || (xhci->xhc_state & XHCI_STATE_HALTED)) {
1518 1519
		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
				"HW died, freeing TD.");
1520
		urb_priv = urb->hcpriv;
1521 1522 1523
		for (i = urb_priv->td_cnt;
		     i < urb_priv->length && xhci->devs[urb->dev->slot_id];
		     i++) {
1524 1525 1526 1527 1528 1529
			td = urb_priv->td[i];
			if (!list_empty(&td->td_list))
				list_del_init(&td->td_list);
			if (!list_empty(&td->cancelled_td_list))
				list_del_init(&td->cancelled_td_list);
		}
1530 1531 1532

		usb_hcd_unlink_urb_from_ep(hcd, urb);
		spin_unlock_irqrestore(&xhci->lock, flags);
1533
		usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1534
		xhci_urb_free_priv(urb_priv);
1535 1536
		return ret;
	}
1537 1538
	if ((xhci->xhc_state & XHCI_STATE_DYING) ||
			(xhci->xhc_state & XHCI_STATE_HALTED)) {
1539 1540 1541
		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
				"Ep 0x%x: URB %p to be canceled on "
				"non-responsive xHCI host.",
1542 1543 1544 1545 1546 1547 1548 1549
				urb->ep->desc.bEndpointAddress, urb);
		/* Let the stop endpoint command watchdog timer (which set this
		 * state) finish cleaning up the endpoint TD lists.  We must
		 * have caught it in the middle of dropping a lock and giving
		 * back an URB.
		 */
		goto done;
	}
1550 1551

	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1552
	ep = &xhci->devs[urb->dev->slot_id]->eps[ep_index];
1553 1554 1555 1556 1557 1558
	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
	if (!ep_ring) {
		ret = -EINVAL;
		goto done;
	}

1559
	urb_priv = urb->hcpriv;
1560 1561
	i = urb_priv->td_cnt;
	if (i < urb_priv->length)
1562 1563 1564
		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
				"Cancel URB %p, dev %s, ep 0x%x, "
				"starting at offset 0x%llx",
1565 1566 1567 1568 1569 1570 1571
				urb, urb->dev->devpath,
				urb->ep->desc.bEndpointAddress,
				(unsigned long long) xhci_trb_virt_to_dma(
					urb_priv->td[i]->start_seg,
					urb_priv->td[i]->first_trb));

	for (; i < urb_priv->length; i++) {
1572 1573 1574 1575
		td = urb_priv->td[i];
		list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
	}

1576 1577 1578
	/* Queue a stop endpoint command, but only if this is
	 * the first cancellation to be handled.
	 */
1579
	if (!(ep->ep_state & EP_HALT_PENDING)) {
1580
		command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1581 1582 1583 1584
		if (!command) {
			ret = -ENOMEM;
			goto done;
		}
1585
		ep->ep_state |= EP_HALT_PENDING;
1586 1587 1588 1589
		ep->stop_cmds_pending++;
		ep->stop_cmd_timer.expires = jiffies +
			XHCI_STOP_EP_CMD_TIMEOUT * HZ;
		add_timer(&ep->stop_cmd_timer);
1590 1591
		xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
					 ep_index, 0);
1592
		xhci_ring_cmd_db(xhci);
1593 1594 1595 1596
	}
done:
	spin_unlock_irqrestore(&xhci->lock, flags);
	return ret;
1597 1598
}

1599 1600 1601 1602 1603 1604 1605 1606
/* Drop an endpoint from a new bandwidth configuration for this device.
 * Only one call to this function is allowed per endpoint before
 * check_bandwidth() or reset_bandwidth() must be called.
 * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
 * add the endpoint to the schedule with possibly new parameters denoted by a
 * different endpoint descriptor in usb_host_endpoint.
 * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
 * not allowed.
1607 1608 1609 1610
 *
 * The USB core will not allow URBs to be queued to an endpoint that is being
 * disabled, so there's no need for mutual exclusion to protect
 * the xhci->devs[slot_id] structure.
1611 1612 1613 1614 1615
 */
int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
		struct usb_host_endpoint *ep)
{
	struct xhci_hcd *xhci;
1616 1617
	struct xhci_container_ctx *in_ctx, *out_ctx;
	struct xhci_input_control_ctx *ctrl_ctx;
1618 1619 1620
	unsigned int ep_index;
	struct xhci_ep_ctx *ep_ctx;
	u32 drop_flag;
1621
	u32 new_add_flags, new_drop_flags;
1622 1623
	int ret;

1624
	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1625 1626 1627
	if (ret <= 0)
		return ret;
	xhci = hcd_to_xhci(hcd);
1628 1629
	if (xhci->xhc_state & XHCI_STATE_DYING)
		return -ENODEV;
1630

1631
	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1632 1633 1634 1635 1636 1637 1638 1639
	drop_flag = xhci_get_endpoint_flag(&ep->desc);
	if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
		xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
				__func__, drop_flag);
		return 0;
	}

	in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1640
	out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1641
	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1642 1643 1644 1645 1646 1647
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return 0;
	}

1648
	ep_index = xhci_get_endpoint_index(&ep->desc);
1649
	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1650 1651 1652
	/* If the HC already knows the endpoint is disabled,
	 * or the HCD has noted it is disabled, ignore this request
	 */
1653
	if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
M
Matt Evans 已提交
1654 1655
	    le32_to_cpu(ctrl_ctx->drop_flags) &
	    xhci_get_endpoint_flag(&ep->desc)) {
1656 1657 1658 1659
		/* Do not warn when called after a usb_device_reset */
		if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
			xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
				  __func__, ep);
1660 1661 1662
		return 0;
	}

M
Matt Evans 已提交
1663 1664
	ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1665

M
Matt Evans 已提交
1666 1667
	ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1668 1669 1670

	xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);

1671 1672 1673
	if (xhci->quirks & XHCI_MTK_HOST)
		xhci_mtk_drop_ep_quirk(hcd, udev, ep);

1674
	xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1675 1676 1677
			(unsigned int) ep->desc.bEndpointAddress,
			udev->slot_id,
			(unsigned int) new_drop_flags,
1678
			(unsigned int) new_add_flags);
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
	return 0;
}

/* Add an endpoint to a new possible bandwidth configuration for this device.
 * Only one call to this function is allowed per endpoint before
 * check_bandwidth() or reset_bandwidth() must be called.
 * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
 * add the endpoint to the schedule with possibly new parameters denoted by a
 * different endpoint descriptor in usb_host_endpoint.
 * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
 * not allowed.
1690 1691 1692 1693
 *
 * The USB core will not allow URBs to be queued to an endpoint until the
 * configuration or alt setting is installed in the device, so there's no need
 * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1694 1695 1696 1697 1698
 */
int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
		struct usb_host_endpoint *ep)
{
	struct xhci_hcd *xhci;
1699
	struct xhci_container_ctx *in_ctx;
1700
	unsigned int ep_index;
1701
	struct xhci_input_control_ctx *ctrl_ctx;
1702
	u32 added_ctxs;
1703
	u32 new_add_flags, new_drop_flags;
1704
	struct xhci_virt_device *virt_dev;
1705 1706
	int ret = 0;

1707
	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1708 1709 1710
	if (ret <= 0) {
		/* So we won't queue a reset ep command for a root hub */
		ep->hcpriv = NULL;
1711
		return ret;
1712
	}
1713
	xhci = hcd_to_xhci(hcd);
1714 1715
	if (xhci->xhc_state & XHCI_STATE_DYING)
		return -ENODEV;
1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727

	added_ctxs = xhci_get_endpoint_flag(&ep->desc);
	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
		/* FIXME when we have to issue an evaluate endpoint command to
		 * deal with ep0 max packet size changing once we get the
		 * descriptors
		 */
		xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
				__func__, added_ctxs);
		return 0;
	}

1728 1729
	virt_dev = xhci->devs[udev->slot_id];
	in_ctx = virt_dev->in_ctx;
1730
	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1731 1732 1733 1734 1735
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return 0;
	}
1736

1737
	ep_index = xhci_get_endpoint_index(&ep->desc);
1738 1739 1740 1741
	/* If this endpoint is already in use, and the upper layers are trying
	 * to add it again without dropping it, reject the addition.
	 */
	if (virt_dev->eps[ep_index].ring &&
1742
			!(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
1743 1744 1745 1746 1747 1748
		xhci_warn(xhci, "Trying to add endpoint 0x%x "
				"without dropping it.\n",
				(unsigned int) ep->desc.bEndpointAddress);
		return -EINVAL;
	}

1749 1750 1751
	/* If the HCD has already noted the endpoint is enabled,
	 * ignore this request.
	 */
1752
	if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
1753 1754
		xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
				__func__, ep);
1755 1756 1757
		return 0;
	}

1758 1759 1760 1761 1762
	/*
	 * Configuration and alternate setting changes must be done in
	 * process context, not interrupt context (or so documenation
	 * for usb_set_interface() and usb_set_configuration() claim).
	 */
1763
	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
1764 1765 1766 1767 1768
		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
				__func__, ep->desc.bEndpointAddress);
		return -ENOMEM;
	}

1769 1770 1771 1772 1773 1774 1775 1776 1777
	if (xhci->quirks & XHCI_MTK_HOST) {
		ret = xhci_mtk_add_ep_quirk(hcd, udev, ep);
		if (ret < 0) {
			xhci_free_or_cache_endpoint_ring(xhci,
				virt_dev, ep_index);
			return ret;
		}
	}

M
Matt Evans 已提交
1778 1779
	ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1780 1781 1782 1783 1784 1785 1786

	/* If xhci_endpoint_disable() was called for this endpoint, but the
	 * xHC hasn't been notified yet through the check_bandwidth() call,
	 * this re-adds a new state for the endpoint from the new endpoint
	 * descriptors.  We must drop and re-add this endpoint, so we leave the
	 * drop flags alone.
	 */
M
Matt Evans 已提交
1787
	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1788

1789 1790 1791
	/* Store the usb_device pointer for later use */
	ep->hcpriv = udev;

1792
	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1793 1794 1795
			(unsigned int) ep->desc.bEndpointAddress,
			udev->slot_id,
			(unsigned int) new_drop_flags,
1796
			(unsigned int) new_add_flags);
1797 1798 1799
	return 0;
}

1800
static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
1801
{
1802
	struct xhci_input_control_ctx *ctrl_ctx;
1803
	struct xhci_ep_ctx *ep_ctx;
1804
	struct xhci_slot_ctx *slot_ctx;
1805 1806
	int i;

1807
	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1808 1809 1810 1811 1812 1813
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return;
	}

1814 1815 1816 1817 1818
	/* When a device's add flag and drop flag are zero, any subsequent
	 * configure endpoint command will leave that endpoint's state
	 * untouched.  Make sure we don't leave any old state in the input
	 * endpoint contexts.
	 */
1819 1820 1821
	ctrl_ctx->drop_flags = 0;
	ctrl_ctx->add_flags = 0;
	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
M
Matt Evans 已提交
1822
	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
1823
	/* Endpoint 0 is always valid */
M
Matt Evans 已提交
1824
	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
1825
	for (i = 1; i < 31; ++i) {
1826
		ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
1827 1828
		ep_ctx->ep_info = 0;
		ep_ctx->ep_info2 = 0;
1829
		ep_ctx->deq = 0;
1830 1831 1832 1833
		ep_ctx->tx_info = 0;
	}
}

1834
static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
1835
		struct usb_device *udev, u32 *cmd_status)
1836 1837 1838
{
	int ret;

1839
	switch (*cmd_status) {
1840 1841 1842 1843 1844
	case COMP_CMD_ABORT:
	case COMP_CMD_STOP:
		xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
		ret = -ETIME;
		break;
1845
	case COMP_ENOMEM:
1846 1847
		dev_warn(&udev->dev,
			 "Not enough host controller resources for new device state.\n");
1848 1849 1850 1851
		ret = -ENOMEM;
		/* FIXME: can we allocate more resources for the HC? */
		break;
	case COMP_BW_ERR:
1852
	case COMP_2ND_BW_ERR:
1853 1854
		dev_warn(&udev->dev,
			 "Not enough bandwidth for new device state.\n");
1855 1856 1857 1858 1859 1860 1861 1862 1863 1864
		ret = -ENOSPC;
		/* FIXME: can we go back to the old state? */
		break;
	case COMP_TRB_ERR:
		/* the HCD set up something wrong */
		dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
				"add flag = 1, "
				"and endpoint is not disabled.\n");
		ret = -EINVAL;
		break;
A
Alex He 已提交
1865
	case COMP_DEV_ERR:
1866 1867
		dev_warn(&udev->dev,
			 "ERROR: Incompatible device for endpoint configure command.\n");
A
Alex He 已提交
1868 1869
		ret = -ENODEV;
		break;
1870
	case COMP_SUCCESS:
1871 1872
		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
				"Successful Endpoint Configure command");
1873 1874 1875
		ret = 0;
		break;
	default:
1876 1877
		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
				*cmd_status);
1878 1879 1880 1881 1882 1883 1884
		ret = -EINVAL;
		break;
	}
	return ret;
}

static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
1885
		struct usb_device *udev, u32 *cmd_status)
1886 1887
{
	int ret;
1888
	struct xhci_virt_device *virt_dev = xhci->devs[udev->slot_id];
1889

1890
	switch (*cmd_status) {
1891 1892 1893 1894 1895
	case COMP_CMD_ABORT:
	case COMP_CMD_STOP:
		xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
		ret = -ETIME;
		break;
1896
	case COMP_EINVAL:
1897 1898
		dev_warn(&udev->dev,
			 "WARN: xHCI driver setup invalid evaluate context command.\n");
1899 1900 1901
		ret = -EINVAL;
		break;
	case COMP_EBADSLT:
1902 1903
		dev_warn(&udev->dev,
			"WARN: slot not enabled for evaluate context command.\n");
1904 1905
		ret = -EINVAL;
		break;
1906
	case COMP_CTX_STATE:
1907 1908
		dev_warn(&udev->dev,
			"WARN: invalid context state for evaluate context command.\n");
1909 1910 1911
		xhci_dbg_ctx(xhci, virt_dev->out_ctx, 1);
		ret = -EINVAL;
		break;
A
Alex He 已提交
1912
	case COMP_DEV_ERR:
1913 1914
		dev_warn(&udev->dev,
			"ERROR: Incompatible device for evaluate context command.\n");
A
Alex He 已提交
1915 1916
		ret = -ENODEV;
		break;
1917 1918 1919 1920 1921
	case COMP_MEL_ERR:
		/* Max Exit Latency too large error */
		dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
		ret = -EINVAL;
		break;
1922
	case COMP_SUCCESS:
1923 1924
		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
				"Successful evaluate context command");
1925 1926 1927
		ret = 0;
		break;
	default:
1928 1929
		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
			*cmd_status);
1930 1931 1932 1933 1934 1935
		ret = -EINVAL;
		break;
	}
	return ret;
}

1936
static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
1937
		struct xhci_input_control_ctx *ctrl_ctx)
1938 1939 1940 1941 1942 1943 1944 1945
{
	u32 valid_add_flags;
	u32 valid_drop_flags;

	/* Ignore the slot flag (bit 0), and the default control endpoint flag
	 * (bit 1).  The default control endpoint is added during the Address
	 * Device command and is never removed until the slot is disabled.
	 */
1946 1947
	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
1948 1949 1950 1951 1952 1953 1954 1955 1956 1957

	/* Use hweight32 to count the number of ones in the add flags, or
	 * number of endpoints added.  Don't count endpoints that are changed
	 * (both added and dropped).
	 */
	return hweight32(valid_add_flags) -
		hweight32(valid_add_flags & valid_drop_flags);
}

static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
1958
		struct xhci_input_control_ctx *ctrl_ctx)
1959 1960 1961 1962
{
	u32 valid_add_flags;
	u32 valid_drop_flags;

1963 1964
	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983

	return hweight32(valid_drop_flags) -
		hweight32(valid_add_flags & valid_drop_flags);
}

/*
 * We need to reserve the new number of endpoints before the configure endpoint
 * command completes.  We can't subtract the dropped endpoints from the number
 * of active endpoints until the command completes because we can oversubscribe
 * the host in this case:
 *
 *  - the first configure endpoint command drops more endpoints than it adds
 *  - a second configure endpoint command that adds more endpoints is queued
 *  - the first configure endpoint command fails, so the config is unchanged
 *  - the second command may succeed, even though there isn't enough resources
 *
 * Must be called with xhci->lock held.
 */
static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
1984
		struct xhci_input_control_ctx *ctrl_ctx)
1985 1986 1987
{
	u32 added_eps;

1988
	added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
1989
	if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
1990 1991 1992
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Not enough ep ctxs: "
				"%u active, need to add %u, limit is %u.",
1993 1994 1995 1996 1997
				xhci->num_active_eps, added_eps,
				xhci->limit_active_eps);
		return -ENOMEM;
	}
	xhci->num_active_eps += added_eps;
1998 1999
	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
			"Adding %u ep ctxs, %u now active.", added_eps,
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
			xhci->num_active_eps);
	return 0;
}

/*
 * The configure endpoint was failed by the xHC for some other reason, so we
 * need to revert the resources that failed configuration would have used.
 *
 * Must be called with xhci->lock held.
 */
static void xhci_free_host_resources(struct xhci_hcd *xhci,
2011
		struct xhci_input_control_ctx *ctrl_ctx)
2012 2013 2014
{
	u32 num_failed_eps;

2015
	num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2016
	xhci->num_active_eps -= num_failed_eps;
2017 2018
	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
			"Removing %u failed ep ctxs, %u now active.",
2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
			num_failed_eps,
			xhci->num_active_eps);
}

/*
 * Now that the command has completed, clean up the active endpoint count by
 * subtracting out the endpoints that were dropped (but not changed).
 *
 * Must be called with xhci->lock held.
 */
static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
2030
		struct xhci_input_control_ctx *ctrl_ctx)
2031 2032 2033
{
	u32 num_dropped_eps;

2034
	num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
2035 2036
	xhci->num_active_eps -= num_dropped_eps;
	if (num_dropped_eps)
2037 2038
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Removing %u dropped ep ctxs, %u now active.",
2039 2040 2041 2042
				num_dropped_eps,
				xhci->num_active_eps);
}

F
Felipe Balbi 已提交
2043
static unsigned int xhci_get_block_size(struct usb_device *udev)
2044 2045 2046 2047 2048 2049 2050 2051
{
	switch (udev->speed) {
	case USB_SPEED_LOW:
	case USB_SPEED_FULL:
		return FS_BLOCK;
	case USB_SPEED_HIGH:
		return HS_BLOCK;
	case USB_SPEED_SUPER:
2052
	case USB_SPEED_SUPER_PLUS:
2053 2054 2055 2056 2057 2058 2059 2060 2061
		return SS_BLOCK;
	case USB_SPEED_UNKNOWN:
	case USB_SPEED_WIRELESS:
	default:
		/* Should never happen */
		return 1;
	}
}

F
Felipe Balbi 已提交
2062 2063
static unsigned int
xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105
{
	if (interval_bw->overhead[LS_OVERHEAD_TYPE])
		return LS_OVERHEAD;
	if (interval_bw->overhead[FS_OVERHEAD_TYPE])
		return FS_OVERHEAD;
	return HS_OVERHEAD;
}

/* If we are changing a LS/FS device under a HS hub,
 * make sure (if we are activating a new TT) that the HS bus has enough
 * bandwidth for this new TT.
 */
static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
		struct xhci_virt_device *virt_dev,
		int old_active_eps)
{
	struct xhci_interval_bw_table *bw_table;
	struct xhci_tt_bw_info *tt_info;

	/* Find the bandwidth table for the root port this TT is attached to. */
	bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;
	tt_info = virt_dev->tt_info;
	/* If this TT already had active endpoints, the bandwidth for this TT
	 * has already been added.  Removing all periodic endpoints (and thus
	 * making the TT enactive) will only decrease the bandwidth used.
	 */
	if (old_active_eps)
		return 0;
	if (old_active_eps == 0 && tt_info->active_eps != 0) {
		if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
			return -ENOMEM;
		return 0;
	}
	/* Not sure why we would have no new active endpoints...
	 *
	 * Maybe because of an Evaluate Context change for a hub update or a
	 * control endpoint 0 max packet size change?
	 * FIXME: skip the bandwidth calculation in that case.
	 */
	return 0;
}

S
Sarah Sharp 已提交
2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121
static int xhci_check_ss_bw(struct xhci_hcd *xhci,
		struct xhci_virt_device *virt_dev)
{
	unsigned int bw_reserved;

	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
	if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
		return -ENOMEM;

	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
	if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
		return -ENOMEM;

	return 0;
}

2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161
/*
 * This algorithm is a very conservative estimate of the worst-case scheduling
 * scenario for any one interval.  The hardware dynamically schedules the
 * packets, so we can't tell which microframe could be the limiting factor in
 * the bandwidth scheduling.  This only takes into account periodic endpoints.
 *
 * Obviously, we can't solve an NP complete problem to find the minimum worst
 * case scenario.  Instead, we come up with an estimate that is no less than
 * the worst case bandwidth used for any one microframe, but may be an
 * over-estimate.
 *
 * We walk the requirements for each endpoint by interval, starting with the
 * smallest interval, and place packets in the schedule where there is only one
 * possible way to schedule packets for that interval.  In order to simplify
 * this algorithm, we record the largest max packet size for each interval, and
 * assume all packets will be that size.
 *
 * For interval 0, we obviously must schedule all packets for each interval.
 * The bandwidth for interval 0 is just the amount of data to be transmitted
 * (the sum of all max ESIT payload sizes, plus any overhead per packet times
 * the number of packets).
 *
 * For interval 1, we have two possible microframes to schedule those packets
 * in.  For this algorithm, if we can schedule the same number of packets for
 * each possible scheduling opportunity (each microframe), we will do so.  The
 * remaining number of packets will be saved to be transmitted in the gaps in
 * the next interval's scheduling sequence.
 *
 * As we move those remaining packets to be scheduled with interval 2 packets,
 * we have to double the number of remaining packets to transmit.  This is
 * because the intervals are actually powers of 2, and we would be transmitting
 * the previous interval's packets twice in this interval.  We also have to be
 * sure that when we look at the largest max packet size for this interval, we
 * also look at the largest max packet size for the remaining packets and take
 * the greater of the two.
 *
 * The algorithm continues to evenly distribute packets in each scheduling
 * opportunity, and push the remaining packets out, until we get to the last
 * interval.  Then those packets and their associated overhead are just added
 * to the bandwidth used.
2162 2163 2164 2165 2166
 */
static int xhci_check_bw_table(struct xhci_hcd *xhci,
		struct xhci_virt_device *virt_dev,
		int old_active_eps)
{
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
	unsigned int bw_reserved;
	unsigned int max_bandwidth;
	unsigned int bw_used;
	unsigned int block_size;
	struct xhci_interval_bw_table *bw_table;
	unsigned int packet_size = 0;
	unsigned int overhead = 0;
	unsigned int packets_transmitted = 0;
	unsigned int packets_remaining = 0;
	unsigned int i;

2178
	if (virt_dev->udev->speed >= USB_SPEED_SUPER)
S
Sarah Sharp 已提交
2179 2180
		return xhci_check_ss_bw(xhci, virt_dev);

2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199
	if (virt_dev->udev->speed == USB_SPEED_HIGH) {
		max_bandwidth = HS_BW_LIMIT;
		/* Convert percent of bus BW reserved to blocks reserved */
		bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
	} else {
		max_bandwidth = FS_BW_LIMIT;
		bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
	}

	bw_table = virt_dev->bw_table;
	/* We need to translate the max packet size and max ESIT payloads into
	 * the units the hardware uses.
	 */
	block_size = xhci_get_block_size(virt_dev->udev);

	/* If we are manipulating a LS/FS device under a HS hub, double check
	 * that the HS bus has enough bandwidth if we are activing a new TT.
	 */
	if (virt_dev->tt_info) {
2200 2201
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Recalculating BW for rootport %u",
2202 2203 2204 2205 2206 2207
				virt_dev->real_port);
		if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
			xhci_warn(xhci, "Not enough bandwidth on HS bus for "
					"newly activated TT.\n");
			return -ENOMEM;
		}
2208 2209
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Recalculating BW for TT slot %u port %u",
2210 2211 2212
				virt_dev->tt_info->slot_id,
				virt_dev->tt_info->ttport);
	} else {
2213 2214
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Recalculating BW for rootport %u",
2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321
				virt_dev->real_port);
	}

	/* Add in how much bandwidth will be used for interval zero, or the
	 * rounded max ESIT payload + number of packets * largest overhead.
	 */
	bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
		bw_table->interval_bw[0].num_packets *
		xhci_get_largest_overhead(&bw_table->interval_bw[0]);

	for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
		unsigned int bw_added;
		unsigned int largest_mps;
		unsigned int interval_overhead;

		/*
		 * How many packets could we transmit in this interval?
		 * If packets didn't fit in the previous interval, we will need
		 * to transmit that many packets twice within this interval.
		 */
		packets_remaining = 2 * packets_remaining +
			bw_table->interval_bw[i].num_packets;

		/* Find the largest max packet size of this or the previous
		 * interval.
		 */
		if (list_empty(&bw_table->interval_bw[i].endpoints))
			largest_mps = 0;
		else {
			struct xhci_virt_ep *virt_ep;
			struct list_head *ep_entry;

			ep_entry = bw_table->interval_bw[i].endpoints.next;
			virt_ep = list_entry(ep_entry,
					struct xhci_virt_ep, bw_endpoint_list);
			/* Convert to blocks, rounding up */
			largest_mps = DIV_ROUND_UP(
					virt_ep->bw_info.max_packet_size,
					block_size);
		}
		if (largest_mps > packet_size)
			packet_size = largest_mps;

		/* Use the larger overhead of this or the previous interval. */
		interval_overhead = xhci_get_largest_overhead(
				&bw_table->interval_bw[i]);
		if (interval_overhead > overhead)
			overhead = interval_overhead;

		/* How many packets can we evenly distribute across
		 * (1 << (i + 1)) possible scheduling opportunities?
		 */
		packets_transmitted = packets_remaining >> (i + 1);

		/* Add in the bandwidth used for those scheduled packets */
		bw_added = packets_transmitted * (overhead + packet_size);

		/* How many packets do we have remaining to transmit? */
		packets_remaining = packets_remaining % (1 << (i + 1));

		/* What largest max packet size should those packets have? */
		/* If we've transmitted all packets, don't carry over the
		 * largest packet size.
		 */
		if (packets_remaining == 0) {
			packet_size = 0;
			overhead = 0;
		} else if (packets_transmitted > 0) {
			/* Otherwise if we do have remaining packets, and we've
			 * scheduled some packets in this interval, take the
			 * largest max packet size from endpoints with this
			 * interval.
			 */
			packet_size = largest_mps;
			overhead = interval_overhead;
		}
		/* Otherwise carry over packet_size and overhead from the last
		 * time we had a remainder.
		 */
		bw_used += bw_added;
		if (bw_used > max_bandwidth) {
			xhci_warn(xhci, "Not enough bandwidth. "
					"Proposed: %u, Max: %u\n",
				bw_used, max_bandwidth);
			return -ENOMEM;
		}
	}
	/*
	 * Ok, we know we have some packets left over after even-handedly
	 * scheduling interval 15.  We don't know which microframes they will
	 * fit into, so we over-schedule and say they will be scheduled every
	 * microframe.
	 */
	if (packets_remaining > 0)
		bw_used += overhead + packet_size;

	if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
		unsigned int port_index = virt_dev->real_port - 1;

		/* OK, we're manipulating a HS device attached to a
		 * root port bandwidth domain.  Include the number of active TTs
		 * in the bandwidth used.
		 */
		bw_used += TT_HS_OVERHEAD *
			xhci->rh_bw[port_index].num_active_tts;
	}

2322 2323 2324
	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
		"Final bandwidth: %u, Limit: %u, Reserved: %u, "
		"Available: %u " "percent",
2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
		bw_used, max_bandwidth, bw_reserved,
		(max_bandwidth - bw_used - bw_reserved) * 100 /
		max_bandwidth);

	bw_used += bw_reserved;
	if (bw_used > max_bandwidth) {
		xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
				bw_used, max_bandwidth);
		return -ENOMEM;
	}

	bw_table->bw_used = bw_used;
2337 2338 2339 2340 2341 2342 2343 2344 2345 2346
	return 0;
}

static bool xhci_is_async_ep(unsigned int ep_type)
{
	return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
					ep_type != ISOC_IN_EP &&
					ep_type != INT_IN_EP);
}

S
Sarah Sharp 已提交
2347 2348
static bool xhci_is_sync_in_ep(unsigned int ep_type)
{
2349
	return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
S
Sarah Sharp 已提交
2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365
}

static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
{
	unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);

	if (ep_bw->ep_interval == 0)
		return SS_OVERHEAD_BURST +
			(ep_bw->mult * ep_bw->num_packets *
					(SS_OVERHEAD + mps));
	return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
				(SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
				1 << ep_bw->ep_interval);

}

2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
		struct xhci_bw_info *ep_bw,
		struct xhci_interval_bw_table *bw_table,
		struct usb_device *udev,
		struct xhci_virt_ep *virt_ep,
		struct xhci_tt_bw_info *tt_info)
{
	struct xhci_interval_bw	*interval_bw;
	int normalized_interval;

S
Sarah Sharp 已提交
2376
	if (xhci_is_async_ep(ep_bw->type))
2377 2378
		return;

2379
	if (udev->speed >= USB_SPEED_SUPER) {
S
Sarah Sharp 已提交
2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
		if (xhci_is_sync_in_ep(ep_bw->type))
			xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
				xhci_get_ss_bw_consumed(ep_bw);
		else
			xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
				xhci_get_ss_bw_consumed(ep_bw);
		return;
	}

	/* SuperSpeed endpoints never get added to intervals in the table, so
	 * this check is only valid for HS/FS/LS devices.
	 */
	if (list_empty(&virt_ep->bw_endpoint_list))
		return;
2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416
	/* For LS/FS devices, we need to translate the interval expressed in
	 * microframes to frames.
	 */
	if (udev->speed == USB_SPEED_HIGH)
		normalized_interval = ep_bw->ep_interval;
	else
		normalized_interval = ep_bw->ep_interval - 3;

	if (normalized_interval == 0)
		bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
	interval_bw = &bw_table->interval_bw[normalized_interval];
	interval_bw->num_packets -= ep_bw->num_packets;
	switch (udev->speed) {
	case USB_SPEED_LOW:
		interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
		break;
	case USB_SPEED_FULL:
		interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
		break;
	case USB_SPEED_HIGH:
		interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
		break;
	case USB_SPEED_SUPER:
2417
	case USB_SPEED_SUPER_PLUS:
2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443
	case USB_SPEED_UNKNOWN:
	case USB_SPEED_WIRELESS:
		/* Should never happen because only LS/FS/HS endpoints will get
		 * added to the endpoint list.
		 */
		return;
	}
	if (tt_info)
		tt_info->active_eps -= 1;
	list_del_init(&virt_ep->bw_endpoint_list);
}

static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
		struct xhci_bw_info *ep_bw,
		struct xhci_interval_bw_table *bw_table,
		struct usb_device *udev,
		struct xhci_virt_ep *virt_ep,
		struct xhci_tt_bw_info *tt_info)
{
	struct xhci_interval_bw	*interval_bw;
	struct xhci_virt_ep *smaller_ep;
	int normalized_interval;

	if (xhci_is_async_ep(ep_bw->type))
		return;

S
Sarah Sharp 已提交
2444 2445 2446 2447 2448 2449 2450 2451 2452 2453
	if (udev->speed == USB_SPEED_SUPER) {
		if (xhci_is_sync_in_ep(ep_bw->type))
			xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
				xhci_get_ss_bw_consumed(ep_bw);
		else
			xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
				xhci_get_ss_bw_consumed(ep_bw);
		return;
	}

2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476
	/* For LS/FS devices, we need to translate the interval expressed in
	 * microframes to frames.
	 */
	if (udev->speed == USB_SPEED_HIGH)
		normalized_interval = ep_bw->ep_interval;
	else
		normalized_interval = ep_bw->ep_interval - 3;

	if (normalized_interval == 0)
		bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
	interval_bw = &bw_table->interval_bw[normalized_interval];
	interval_bw->num_packets += ep_bw->num_packets;
	switch (udev->speed) {
	case USB_SPEED_LOW:
		interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
		break;
	case USB_SPEED_FULL:
		interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
		break;
	case USB_SPEED_HIGH:
		interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
		break;
	case USB_SPEED_SUPER:
2477
	case USB_SPEED_SUPER_PLUS:
2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
	case USB_SPEED_UNKNOWN:
	case USB_SPEED_WIRELESS:
		/* Should never happen because only LS/FS/HS endpoints will get
		 * added to the endpoint list.
		 */
		return;
	}

	if (tt_info)
		tt_info->active_eps += 1;
	/* Insert the endpoint into the list, largest max packet size first. */
	list_for_each_entry(smaller_ep, &interval_bw->endpoints,
			bw_endpoint_list) {
		if (ep_bw->max_packet_size >=
				smaller_ep->bw_info.max_packet_size) {
			/* Add the new ep before the smaller endpoint */
			list_add_tail(&virt_ep->bw_endpoint_list,
					&smaller_ep->bw_endpoint_list);
			return;
		}
	}
	/* Add the new endpoint at the end of the list. */
	list_add_tail(&virt_ep->bw_endpoint_list,
			&interval_bw->endpoints);
}

void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
		struct xhci_virt_device *virt_dev,
		int old_active_eps)
{
	struct xhci_root_port_bw_info *rh_bw_info;
	if (!virt_dev->tt_info)
		return;

	rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];
	if (old_active_eps == 0 &&
				virt_dev->tt_info->active_eps != 0) {
		rh_bw_info->num_active_tts += 1;
2516
		rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2517 2518 2519
	} else if (old_active_eps != 0 &&
				virt_dev->tt_info->active_eps == 0) {
		rh_bw_info->num_active_tts -= 1;
2520
		rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535
	}
}

static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
		struct xhci_virt_device *virt_dev,
		struct xhci_container_ctx *in_ctx)
{
	struct xhci_bw_info ep_bw_info[31];
	int i;
	struct xhci_input_control_ctx *ctrl_ctx;
	int old_active_eps = 0;

	if (virt_dev->tt_info)
		old_active_eps = virt_dev->tt_info->active_eps;

2536
	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2537 2538 2539 2540 2541
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return -ENOMEM;
	}
2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613

	for (i = 0; i < 31; i++) {
		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
			continue;

		/* Make a copy of the BW info in case we need to revert this */
		memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
				sizeof(ep_bw_info[i]));
		/* Drop the endpoint from the interval table if the endpoint is
		 * being dropped or changed.
		 */
		if (EP_IS_DROPPED(ctrl_ctx, i))
			xhci_drop_ep_from_interval_table(xhci,
					&virt_dev->eps[i].bw_info,
					virt_dev->bw_table,
					virt_dev->udev,
					&virt_dev->eps[i],
					virt_dev->tt_info);
	}
	/* Overwrite the information stored in the endpoints' bw_info */
	xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
	for (i = 0; i < 31; i++) {
		/* Add any changed or added endpoints to the interval table */
		if (EP_IS_ADDED(ctrl_ctx, i))
			xhci_add_ep_to_interval_table(xhci,
					&virt_dev->eps[i].bw_info,
					virt_dev->bw_table,
					virt_dev->udev,
					&virt_dev->eps[i],
					virt_dev->tt_info);
	}

	if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
		/* Ok, this fits in the bandwidth we have.
		 * Update the number of active TTs.
		 */
		xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
		return 0;
	}

	/* We don't have enough bandwidth for this, revert the stored info. */
	for (i = 0; i < 31; i++) {
		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
			continue;

		/* Drop the new copies of any added or changed endpoints from
		 * the interval table.
		 */
		if (EP_IS_ADDED(ctrl_ctx, i)) {
			xhci_drop_ep_from_interval_table(xhci,
					&virt_dev->eps[i].bw_info,
					virt_dev->bw_table,
					virt_dev->udev,
					&virt_dev->eps[i],
					virt_dev->tt_info);
		}
		/* Revert the endpoint back to its old information */
		memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
				sizeof(ep_bw_info[i]));
		/* Add any changed or dropped endpoints back into the table */
		if (EP_IS_DROPPED(ctrl_ctx, i))
			xhci_add_ep_to_interval_table(xhci,
					&virt_dev->eps[i].bw_info,
					virt_dev->bw_table,
					virt_dev->udev,
					&virt_dev->eps[i],
					virt_dev->tt_info);
	}
	return -ENOMEM;
}


2614 2615 2616 2617
/* Issue a configure endpoint command or evaluate context command
 * and wait for it to finish.
 */
static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2618 2619 2620
		struct usb_device *udev,
		struct xhci_command *command,
		bool ctx_change, bool must_succeed)
2621 2622 2623
{
	int ret;
	unsigned long flags;
2624
	struct xhci_input_control_ctx *ctrl_ctx;
2625
	struct xhci_virt_device *virt_dev;
2626 2627 2628

	if (!command)
		return -EINVAL;
2629 2630

	spin_lock_irqsave(&xhci->lock, flags);
2631
	virt_dev = xhci->devs[udev->slot_id];
2632

2633
	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2634
	if (!ctrl_ctx) {
2635
		spin_unlock_irqrestore(&xhci->lock, flags);
2636 2637 2638 2639
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return -ENOMEM;
	}
2640

2641
	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2642
			xhci_reserve_host_resources(xhci, ctrl_ctx)) {
2643 2644 2645 2646 2647 2648
		spin_unlock_irqrestore(&xhci->lock, flags);
		xhci_warn(xhci, "Not enough host resources, "
				"active endpoint contexts = %u\n",
				xhci->num_active_eps);
		return -ENOMEM;
	}
2649
	if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
2650
	    xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
2651
		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2652
			xhci_free_host_resources(xhci, ctrl_ctx);
2653 2654 2655 2656
		spin_unlock_irqrestore(&xhci->lock, flags);
		xhci_warn(xhci, "Not enough bandwidth\n");
		return -ENOMEM;
	}
2657

2658
	if (!ctx_change)
2659 2660
		ret = xhci_queue_configure_endpoint(xhci, command,
				command->in_ctx->dma,
2661
				udev->slot_id, must_succeed);
2662
	else
2663 2664
		ret = xhci_queue_evaluate_context(xhci, command,
				command->in_ctx->dma,
2665
				udev->slot_id, must_succeed);
2666
	if (ret < 0) {
2667
		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2668
			xhci_free_host_resources(xhci, ctrl_ctx);
2669
		spin_unlock_irqrestore(&xhci->lock, flags);
2670 2671
		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
				"FIXME allocate a new ring segment");
2672 2673 2674 2675 2676 2677
		return -ENOMEM;
	}
	xhci_ring_cmd_db(xhci);
	spin_unlock_irqrestore(&xhci->lock, flags);

	/* Wait for the configure endpoint command to complete */
2678
	wait_for_completion(command->completion);
2679 2680

	if (!ctx_change)
2681 2682
		ret = xhci_configure_endpoint_result(xhci, udev,
						     &command->status);
2683
	else
2684 2685
		ret = xhci_evaluate_context_result(xhci, udev,
						   &command->status);
2686 2687 2688 2689 2690 2691 2692

	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
		spin_lock_irqsave(&xhci->lock, flags);
		/* If the command failed, remove the reserved resources.
		 * Otherwise, clean up the estimate to include dropped eps.
		 */
		if (ret)
2693
			xhci_free_host_resources(xhci, ctrl_ctx);
2694
		else
2695
			xhci_finish_resource_reservation(xhci, ctrl_ctx);
2696 2697 2698
		spin_unlock_irqrestore(&xhci->lock, flags);
	}
	return ret;
2699 2700
}

2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714
static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
	struct xhci_virt_device *vdev, int i)
{
	struct xhci_virt_ep *ep = &vdev->eps[i];

	if (ep->ep_state & EP_HAS_STREAMS) {
		xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
				xhci_get_endpoint_address(i));
		xhci_free_stream_info(xhci, ep->stream_info);
		ep->stream_info = NULL;
		ep->ep_state &= ~EP_HAS_STREAMS;
	}
}

2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
/* Called after one or more calls to xhci_add_endpoint() or
 * xhci_drop_endpoint().  If this call fails, the USB core is expected
 * to call xhci_reset_bandwidth().
 *
 * Since we are in the middle of changing either configuration or
 * installing a new alt setting, the USB core won't allow URBs to be
 * enqueued for any endpoint on the old config or interface.  Nothing
 * else should be touching the xhci->devs[slot_id] structure, so we
 * don't need to take the xhci->lock for manipulating that.
 */
2725 2726 2727 2728 2729 2730
int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
{
	int i;
	int ret = 0;
	struct xhci_hcd *xhci;
	struct xhci_virt_device	*virt_dev;
2731 2732
	struct xhci_input_control_ctx *ctrl_ctx;
	struct xhci_slot_ctx *slot_ctx;
2733
	struct xhci_command *command;
2734

2735
	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2736 2737 2738
	if (ret <= 0)
		return ret;
	xhci = hcd_to_xhci(hcd);
2739 2740
	if ((xhci->xhc_state & XHCI_STATE_DYING) ||
		(xhci->xhc_state & XHCI_STATE_REMOVING))
2741
		return -ENODEV;
2742

2743
	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2744 2745
	virt_dev = xhci->devs[udev->slot_id];

2746 2747 2748 2749 2750 2751
	command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
	if (!command)
		return -ENOMEM;

	command->in_ctx = virt_dev->in_ctx;

2752
	/* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
2753
	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2754 2755 2756
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
2757 2758
		ret = -ENOMEM;
		goto command_cleanup;
2759
	}
M
Matt Evans 已提交
2760 2761 2762
	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
2763 2764 2765

	/* Don't issue the command if there's no endpoints to update. */
	if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
2766 2767 2768 2769
	    ctrl_ctx->drop_flags == 0) {
		ret = 0;
		goto command_cleanup;
	}
2770
	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
2771
	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782
	for (i = 31; i >= 1; i--) {
		__le32 le32 = cpu_to_le32(BIT(i));

		if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
		    || (ctrl_ctx->add_flags & le32) || i == 1) {
			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
			break;
		}
	}
	xhci_dbg(xhci, "New Input Control Context:\n");
2783
	xhci_dbg_ctx(xhci, virt_dev->in_ctx,
M
Matt Evans 已提交
2784
		     LAST_CTX_TO_EP_NUM(le32_to_cpu(slot_ctx->dev_info)));
2785

2786
	ret = xhci_configure_endpoint(xhci, udev, command,
2787
			false, false);
2788
	if (ret)
2789
		/* Callee should call reset_bandwidth() */
2790
		goto command_cleanup;
2791 2792

	xhci_dbg(xhci, "Output context after successful config ep cmd:\n");
2793
	xhci_dbg_ctx(xhci, virt_dev->out_ctx,
M
Matt Evans 已提交
2794
		     LAST_CTX_TO_EP_NUM(le32_to_cpu(slot_ctx->dev_info)));
2795

2796 2797
	/* Free any rings that were dropped, but not changed. */
	for (i = 1; i < 31; ++i) {
2798
		if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
2799
		    !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
2800
			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
2801 2802
			xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
		}
2803
	}
2804
	xhci_zero_in_ctx(xhci, virt_dev);
2805 2806 2807 2808
	/*
	 * Install any rings for completely new endpoints or changed endpoints,
	 * and free or cache any old rings from changed endpoints.
	 */
2809
	for (i = 1; i < 31; ++i) {
2810 2811 2812 2813 2814 2815
		if (!virt_dev->eps[i].new_ring)
			continue;
		/* Only cache or free the old ring if it exists.
		 * It may not if this is the first add of an endpoint.
		 */
		if (virt_dev->eps[i].ring) {
2816
			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
2817
		}
2818
		xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
2819 2820
		virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
		virt_dev->eps[i].new_ring = NULL;
2821
	}
2822 2823 2824
command_cleanup:
	kfree(command->completion);
	kfree(command);
2825 2826 2827 2828 2829 2830 2831 2832 2833 2834

	return ret;
}

void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
{
	struct xhci_hcd *xhci;
	struct xhci_virt_device	*virt_dev;
	int i, ret;

2835
	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2836 2837 2838 2839
	if (ret <= 0)
		return;
	xhci = hcd_to_xhci(hcd);

2840
	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2841 2842 2843
	virt_dev = xhci->devs[udev->slot_id];
	/* Free any rings allocated for added endpoints */
	for (i = 0; i < 31; ++i) {
2844 2845 2846
		if (virt_dev->eps[i].new_ring) {
			xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
			virt_dev->eps[i].new_ring = NULL;
2847 2848
		}
	}
2849
	xhci_zero_in_ctx(xhci, virt_dev);
2850 2851
}

2852
static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
2853 2854
		struct xhci_container_ctx *in_ctx,
		struct xhci_container_ctx *out_ctx,
2855
		struct xhci_input_control_ctx *ctrl_ctx,
2856
		u32 add_flags, u32 drop_flags)
2857
{
M
Matt Evans 已提交
2858 2859
	ctrl_ctx->add_flags = cpu_to_le32(add_flags);
	ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
2860
	xhci_slot_copy(xhci, in_ctx, out_ctx);
M
Matt Evans 已提交
2861
	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2862

2863 2864
	xhci_dbg(xhci, "Input Context:\n");
	xhci_dbg_ctx(xhci, in_ctx, xhci_last_valid_endpoint(add_flags));
2865 2866
}

2867
static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
2868 2869 2870
		unsigned int slot_id, unsigned int ep_index,
		struct xhci_dequeue_state *deq_state)
{
2871
	struct xhci_input_control_ctx *ctrl_ctx;
2872 2873 2874 2875 2876
	struct xhci_container_ctx *in_ctx;
	struct xhci_ep_ctx *ep_ctx;
	u32 added_ctxs;
	dma_addr_t addr;

2877
	in_ctx = xhci->devs[slot_id]->in_ctx;
2878
	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2879 2880 2881 2882 2883 2884
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return;
	}

2885 2886
	xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
			xhci->devs[slot_id]->out_ctx, ep_index);
2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897
	ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
	addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
			deq_state->new_deq_ptr);
	if (addr == 0) {
		xhci_warn(xhci, "WARN Cannot submit config ep after "
				"reset ep command\n");
		xhci_warn(xhci, "WARN deq seg = %p, deq ptr = %p\n",
				deq_state->new_deq_seg,
				deq_state->new_deq_ptr);
		return;
	}
M
Matt Evans 已提交
2898
	ep_ctx->deq = cpu_to_le64(addr | deq_state->new_cycle_state);
2899 2900

	added_ctxs = xhci_get_endpoint_flag_from_index(ep_index);
2901
	xhci_setup_input_ctx_for_config_ep(xhci, xhci->devs[slot_id]->in_ctx,
2902 2903
			xhci->devs[slot_id]->out_ctx, ctrl_ctx,
			added_ctxs, added_ctxs);
2904 2905
}

2906
void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci,
2907
			unsigned int ep_index, struct xhci_td *td)
2908 2909
{
	struct xhci_dequeue_state deq_state;
2910
	struct xhci_virt_ep *ep;
2911
	struct usb_device *udev = td->urb->dev;
2912

2913 2914
	xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
			"Cleaning up stalled endpoint ring");
2915
	ep = &xhci->devs[udev->slot_id]->eps[ep_index];
2916 2917 2918 2919
	/* We need to move the HW's dequeue pointer past this TD,
	 * or it will attempt to resend it on the next doorbell ring.
	 */
	xhci_find_new_dequeue_state(xhci, udev->slot_id,
2920
			ep_index, ep->stopped_stream, td, &deq_state);
2921

2922 2923 2924
	if (!deq_state.new_deq_ptr || !deq_state.new_deq_seg)
		return;

2925 2926 2927 2928
	/* HW with the reset endpoint quirk will use the saved dequeue state to
	 * issue a configure endpoint command later.
	 */
	if (!(xhci->quirks & XHCI_RESET_EP_QUIRK)) {
2929 2930
		xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
				"Queueing new dequeue state");
2931
		xhci_queue_new_dequeue_state(xhci, udev->slot_id,
2932
				ep_index, ep->stopped_stream, &deq_state);
2933 2934 2935
	} else {
		/* Better hope no one uses the input context between now and the
		 * reset endpoint completion!
2936 2937
		 * XXX: No idea how this hardware will react when stream rings
		 * are enabled.
2938
		 */
2939 2940 2941
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Setting up input context for "
				"configure endpoint command");
2942 2943 2944
		xhci_setup_input_ctx_for_quirk(xhci, udev->slot_id,
				ep_index, &deq_state);
	}
2945 2946
}

2947
/* Called when clearing halted device. The core should have sent the control
2948
 * message to clear the device halt condition. The host side of the halt should
2949 2950 2951 2952
 * already be cleared with a reset endpoint command issued when the STALL tx
 * event was received.
 *
 * Context: in_interrupt
2953
 */
2954

2955 2956 2957 2958 2959 2960
void xhci_endpoint_reset(struct usb_hcd *hcd,
		struct usb_host_endpoint *ep)
{
	struct xhci_hcd *xhci;

	xhci = hcd_to_xhci(hcd);
2961

2962
	/*
2963
	 * We might need to implement the config ep cmd in xhci 4.8.1 note:
2964 2965 2966 2967 2968
	 * The Reset Endpoint Command may only be issued to endpoints in the
	 * Halted state. If software wishes reset the Data Toggle or Sequence
	 * Number of an endpoint that isn't in the Halted state, then software
	 * may issue a Configure Endpoint Command with the Drop and Add bits set
	 * for the target endpoint. that is in the Stopped state.
2969
	 */
2970

2971 2972 2973
	/* For now just print debug to follow the situation */
	xhci_dbg(xhci, "Endpoint 0x%x ep reset callback called\n",
		 ep->desc.bEndpointAddress);
2974 2975
}

2976 2977 2978 2979 2980 2981 2982 2983 2984 2985
static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
		struct usb_device *udev, struct usb_host_endpoint *ep,
		unsigned int slot_id)
{
	int ret;
	unsigned int ep_index;
	unsigned int ep_state;

	if (!ep)
		return -EINVAL;
2986
	ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
2987 2988
	if (ret <= 0)
		return -EINVAL;
2989
	if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057
		xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
				" descriptor for ep 0x%x does not support streams\n",
				ep->desc.bEndpointAddress);
		return -EINVAL;
	}

	ep_index = xhci_get_endpoint_index(&ep->desc);
	ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
	if (ep_state & EP_HAS_STREAMS ||
			ep_state & EP_GETTING_STREAMS) {
		xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
				"already has streams set up.\n",
				ep->desc.bEndpointAddress);
		xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
				"dynamic stream context array reallocation.\n");
		return -EINVAL;
	}
	if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
		xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
				"endpoint 0x%x; URBs are pending.\n",
				ep->desc.bEndpointAddress);
		return -EINVAL;
	}
	return 0;
}

static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
		unsigned int *num_streams, unsigned int *num_stream_ctxs)
{
	unsigned int max_streams;

	/* The stream context array size must be a power of two */
	*num_stream_ctxs = roundup_pow_of_two(*num_streams);
	/*
	 * Find out how many primary stream array entries the host controller
	 * supports.  Later we may use secondary stream arrays (similar to 2nd
	 * level page entries), but that's an optional feature for xHCI host
	 * controllers. xHCs must support at least 4 stream IDs.
	 */
	max_streams = HCC_MAX_PSA(xhci->hcc_params);
	if (*num_stream_ctxs > max_streams) {
		xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
				max_streams);
		*num_stream_ctxs = max_streams;
		*num_streams = max_streams;
	}
}

/* Returns an error code if one of the endpoint already has streams.
 * This does not change any data structures, it only checks and gathers
 * information.
 */
static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
		struct usb_device *udev,
		struct usb_host_endpoint **eps, unsigned int num_eps,
		unsigned int *num_streams, u32 *changed_ep_bitmask)
{
	unsigned int max_streams;
	unsigned int endpoint_flag;
	int i;
	int ret;

	for (i = 0; i < num_eps; i++) {
		ret = xhci_check_streams_endpoint(xhci, udev,
				eps[i], udev->slot_id);
		if (ret < 0)
			return ret;

3058
		max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093
		if (max_streams < (*num_streams - 1)) {
			xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
					eps[i]->desc.bEndpointAddress,
					max_streams);
			*num_streams = max_streams+1;
		}

		endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
		if (*changed_ep_bitmask & endpoint_flag)
			return -EINVAL;
		*changed_ep_bitmask |= endpoint_flag;
	}
	return 0;
}

static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
		struct usb_device *udev,
		struct usb_host_endpoint **eps, unsigned int num_eps)
{
	u32 changed_ep_bitmask = 0;
	unsigned int slot_id;
	unsigned int ep_index;
	unsigned int ep_state;
	int i;

	slot_id = udev->slot_id;
	if (!xhci->devs[slot_id])
		return 0;

	for (i = 0; i < num_eps; i++) {
		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
		ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
		/* Are streams already being freed for the endpoint? */
		if (ep_state & EP_GETTING_NO_STREAMS) {
			xhci_warn(xhci, "WARN Can't disable streams for "
J
Joe Perches 已提交
3094 3095
					"endpoint 0x%x, "
					"streams are being disabled already\n",
3096 3097 3098 3099 3100 3101 3102
					eps[i]->desc.bEndpointAddress);
			return 0;
		}
		/* Are there actually any streams to free? */
		if (!(ep_state & EP_HAS_STREAMS) &&
				!(ep_state & EP_GETTING_STREAMS)) {
			xhci_warn(xhci, "WARN Can't disable streams for "
J
Joe Perches 已提交
3103 3104
					"endpoint 0x%x, "
					"streams are already disabled!\n",
3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115
					eps[i]->desc.bEndpointAddress);
			xhci_warn(xhci, "WARN xhci_free_streams() called "
					"with non-streams endpoint\n");
			return 0;
		}
		changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
	}
	return changed_ep_bitmask;
}

/*
3116
 * The USB device drivers use this function (through the HCD interface in USB
3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138
 * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
 * coordinate mass storage command queueing across multiple endpoints (basically
 * a stream ID == a task ID).
 *
 * Setting up streams involves allocating the same size stream context array
 * for each endpoint and issuing a configure endpoint command for all endpoints.
 *
 * Don't allow the call to succeed if one endpoint only supports one stream
 * (which means it doesn't support streams at all).
 *
 * Drivers may get less stream IDs than they asked for, if the host controller
 * hardware or endpoints claim they can't support the number of requested
 * stream IDs.
 */
int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
		struct usb_host_endpoint **eps, unsigned int num_eps,
		unsigned int num_streams, gfp_t mem_flags)
{
	int i, ret;
	struct xhci_hcd *xhci;
	struct xhci_virt_device *vdev;
	struct xhci_command *config_cmd;
3139
	struct xhci_input_control_ctx *ctrl_ctx;
3140 3141
	unsigned int ep_index;
	unsigned int num_stream_ctxs;
3142
	unsigned int max_packet;
3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156
	unsigned long flags;
	u32 changed_ep_bitmask = 0;

	if (!eps)
		return -EINVAL;

	/* Add one to the number of streams requested to account for
	 * stream 0 that is reserved for xHCI usage.
	 */
	num_streams += 1;
	xhci = hcd_to_xhci(hcd);
	xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
			num_streams);

H
Hans de Goede 已提交
3157
	/* MaxPSASize value 0 (2 streams) means streams are not supported */
3158 3159
	if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
			HCC_MAX_PSA(xhci->hcc_params) < 4) {
H
Hans de Goede 已提交
3160 3161 3162 3163
		xhci_dbg(xhci, "xHCI controller does not support streams.\n");
		return -ENOSYS;
	}

3164 3165 3166 3167 3168
	config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
	if (!config_cmd) {
		xhci_dbg(xhci, "Could not allocate xHCI command structure.\n");
		return -ENOMEM;
	}
3169
	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
3170 3171 3172 3173 3174 3175
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		xhci_free_command(xhci, config_cmd);
		return -ENOMEM;
	}
3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196

	/* Check to make sure all endpoints are not already configured for
	 * streams.  While we're at it, find the maximum number of streams that
	 * all the endpoints will support and check for duplicate endpoints.
	 */
	spin_lock_irqsave(&xhci->lock, flags);
	ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
			num_eps, &num_streams, &changed_ep_bitmask);
	if (ret < 0) {
		xhci_free_command(xhci, config_cmd);
		spin_unlock_irqrestore(&xhci->lock, flags);
		return ret;
	}
	if (num_streams <= 1) {
		xhci_warn(xhci, "WARN: endpoints can't handle "
				"more than one stream.\n");
		xhci_free_command(xhci, config_cmd);
		spin_unlock_irqrestore(&xhci->lock, flags);
		return -EINVAL;
	}
	vdev = xhci->devs[udev->slot_id];
L
Lucas De Marchi 已提交
3197
	/* Mark each endpoint as being in transition, so
3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215
	 * xhci_urb_enqueue() will reject all URBs.
	 */
	for (i = 0; i < num_eps; i++) {
		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
		vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
	}
	spin_unlock_irqrestore(&xhci->lock, flags);

	/* Setup internal data structures and allocate HW data structures for
	 * streams (but don't install the HW structures in the input context
	 * until we're sure all memory allocation succeeded).
	 */
	xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
	xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
			num_stream_ctxs, num_streams);

	for (i = 0; i < num_eps; i++) {
		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3216
		max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&eps[i]->desc));
3217 3218
		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
				num_stream_ctxs,
3219 3220
				num_streams,
				max_packet, mem_flags);
3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243
		if (!vdev->eps[ep_index].stream_info)
			goto cleanup;
		/* Set maxPstreams in endpoint context and update deq ptr to
		 * point to stream context array. FIXME
		 */
	}

	/* Set up the input context for a configure endpoint command. */
	for (i = 0; i < num_eps; i++) {
		struct xhci_ep_ctx *ep_ctx;

		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
		ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);

		xhci_endpoint_copy(xhci, config_cmd->in_ctx,
				vdev->out_ctx, ep_index);
		xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
				vdev->eps[ep_index].stream_info);
	}
	/* Tell the HW to drop its old copy of the endpoint context info
	 * and add the updated copy from the input context.
	 */
	xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3244 3245
			vdev->out_ctx, ctrl_ctx,
			changed_ep_bitmask, changed_ep_bitmask);
3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276

	/* Issue and wait for the configure endpoint command */
	ret = xhci_configure_endpoint(xhci, udev, config_cmd,
			false, false);

	/* xHC rejected the configure endpoint command for some reason, so we
	 * leave the old ring intact and free our internal streams data
	 * structure.
	 */
	if (ret < 0)
		goto cleanup;

	spin_lock_irqsave(&xhci->lock, flags);
	for (i = 0; i < num_eps; i++) {
		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
		xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
			 udev->slot_id, ep_index);
		vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
	}
	xhci_free_command(xhci, config_cmd);
	spin_unlock_irqrestore(&xhci->lock, flags);

	/* Subtract 1 for stream 0, which drivers can't use */
	return num_streams - 1;

cleanup:
	/* If it didn't work, free the streams! */
	for (i = 0; i < num_eps; i++) {
		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3277
		vdev->eps[ep_index].stream_info = NULL;
3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302
		/* FIXME Unset maxPstreams in endpoint context and
		 * update deq ptr to point to normal string ring.
		 */
		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
		xhci_endpoint_zero(xhci, vdev, eps[i]);
	}
	xhci_free_command(xhci, config_cmd);
	return -ENOMEM;
}

/* Transition the endpoint from using streams to being a "normal" endpoint
 * without streams.
 *
 * Modify the endpoint context state, submit a configure endpoint command,
 * and free all endpoint rings for streams if that completes successfully.
 */
int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
		struct usb_host_endpoint **eps, unsigned int num_eps,
		gfp_t mem_flags)
{
	int i, ret;
	struct xhci_hcd *xhci;
	struct xhci_virt_device *vdev;
	struct xhci_command *command;
3303
	struct xhci_input_control_ctx *ctrl_ctx;
3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325
	unsigned int ep_index;
	unsigned long flags;
	u32 changed_ep_bitmask;

	xhci = hcd_to_xhci(hcd);
	vdev = xhci->devs[udev->slot_id];

	/* Set up a configure endpoint command to remove the streams rings */
	spin_lock_irqsave(&xhci->lock, flags);
	changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
			udev, eps, num_eps);
	if (changed_ep_bitmask == 0) {
		spin_unlock_irqrestore(&xhci->lock, flags);
		return -EINVAL;
	}

	/* Use the xhci_command structure from the first endpoint.  We may have
	 * allocated too many, but the driver may call xhci_free_streams() for
	 * each endpoint it grouped into one call to xhci_alloc_streams().
	 */
	ep_index = xhci_get_endpoint_index(&eps[0]->desc);
	command = vdev->eps[ep_index].stream_info->free_streams_command;
3326
	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3327
	if (!ctrl_ctx) {
3328
		spin_unlock_irqrestore(&xhci->lock, flags);
3329 3330 3331 3332 3333
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return -EINVAL;
	}

3334 3335 3336 3337 3338 3339 3340 3341 3342 3343
	for (i = 0; i < num_eps; i++) {
		struct xhci_ep_ctx *ep_ctx;

		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
		xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
			EP_GETTING_NO_STREAMS;

		xhci_endpoint_copy(xhci, command->in_ctx,
				vdev->out_ctx, ep_index);
3344
		xhci_setup_no_streams_ep_input_ctx(ep_ctx,
3345 3346 3347
				&vdev->eps[ep_index]);
	}
	xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3348 3349
			vdev->out_ctx, ctrl_ctx,
			changed_ep_bitmask, changed_ep_bitmask);
3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367
	spin_unlock_irqrestore(&xhci->lock, flags);

	/* Issue and wait for the configure endpoint command,
	 * which must succeed.
	 */
	ret = xhci_configure_endpoint(xhci, udev, command,
			false, true);

	/* xHC rejected the configure endpoint command for some reason, so we
	 * leave the streams rings intact.
	 */
	if (ret < 0)
		return ret;

	spin_lock_irqsave(&xhci->lock, flags);
	for (i = 0; i < num_eps; i++) {
		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3368
		vdev->eps[ep_index].stream_info = NULL;
3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379
		/* FIXME Unset maxPstreams in endpoint context and
		 * update deq ptr to point to normal string ring.
		 */
		vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
	}
	spin_unlock_irqrestore(&xhci->lock, flags);

	return 0;
}

3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401
/*
 * Deletes endpoint resources for endpoints that were active before a Reset
 * Device command, or a Disable Slot command.  The Reset Device command leaves
 * the control endpoint intact, whereas the Disable Slot command deletes it.
 *
 * Must be called with xhci->lock held.
 */
void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
	struct xhci_virt_device *virt_dev, bool drop_control_ep)
{
	int i;
	unsigned int num_dropped_eps = 0;
	unsigned int drop_flags = 0;

	for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
		if (virt_dev->eps[i].ring) {
			drop_flags |= 1 << i;
			num_dropped_eps++;
		}
	}
	xhci->num_active_eps -= num_dropped_eps;
	if (num_dropped_eps)
3402 3403 3404
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Dropped %u ep ctxs, flags = 0x%x, "
				"%u now active.",
3405 3406 3407 3408
				num_dropped_eps, drop_flags,
				xhci->num_active_eps);
}

3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420
/*
 * This submits a Reset Device Command, which will set the device state to 0,
 * set the device address to 0, and disable all the endpoints except the default
 * control endpoint.  The USB core should come back and call
 * xhci_address_device(), and then re-set up the configuration.  If this is
 * called because of a usb_reset_and_verify_device(), then the old alternate
 * settings will be re-installed through the normal bandwidth allocation
 * functions.
 *
 * Wait for the Reset Device command to finish.  Remove all structures
 * associated with the endpoints that were disabled.  Clear the input device
 * structure?  Cache the rings?  Reset the control endpoint 0 max packet size?
3421 3422 3423 3424 3425
 *
 * If the virt_dev to be reset does not exist or does not match the udev,
 * it means the device is lost, possibly due to the xHC restore error and
 * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
 * re-allocate the device.
3426
 */
3427
int xhci_discover_or_reset_device(struct usb_hcd *hcd, struct usb_device *udev)
3428 3429 3430 3431 3432 3433 3434 3435
{
	int ret, i;
	unsigned long flags;
	struct xhci_hcd *xhci;
	unsigned int slot_id;
	struct xhci_virt_device *virt_dev;
	struct xhci_command *reset_device_cmd;
	int last_freed_endpoint;
3436
	struct xhci_slot_ctx *slot_ctx;
3437
	int old_active_eps = 0;
3438

3439
	ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3440 3441 3442 3443 3444
	if (ret <= 0)
		return ret;
	xhci = hcd_to_xhci(hcd);
	slot_id = udev->slot_id;
	virt_dev = xhci->devs[slot_id];
3445 3446 3447 3448 3449 3450 3451 3452 3453 3454
	if (!virt_dev) {
		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
				"not exist. Re-allocate the device\n", slot_id);
		ret = xhci_alloc_dev(hcd, udev);
		if (ret == 1)
			return 0;
		else
			return -EINVAL;
	}

3455 3456 3457
	if (virt_dev->tt_info)
		old_active_eps = virt_dev->tt_info->active_eps;

3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471
	if (virt_dev->udev != udev) {
		/* If the virt_dev and the udev does not match, this virt_dev
		 * may belong to another udev.
		 * Re-allocate the device.
		 */
		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
				"not match the udev. Re-allocate the device\n",
				slot_id);
		ret = xhci_alloc_dev(hcd, udev);
		if (ret == 1)
			return 0;
		else
			return -EINVAL;
	}
3472

3473 3474 3475 3476 3477 3478
	/* If device is not setup, there is no point in resetting it */
	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
	if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
						SLOT_STATE_DISABLED)
		return 0;

3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493
	xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
	/* Allocate the command structure that holds the struct completion.
	 * Assume we're in process context, since the normal device reset
	 * process has to wait for the device anyway.  Storage devices are
	 * reset as part of error handling, so use GFP_NOIO instead of
	 * GFP_KERNEL.
	 */
	reset_device_cmd = xhci_alloc_command(xhci, false, true, GFP_NOIO);
	if (!reset_device_cmd) {
		xhci_dbg(xhci, "Couldn't allocate command structure.\n");
		return -ENOMEM;
	}

	/* Attempt to submit the Reset Device command to the command ring */
	spin_lock_irqsave(&xhci->lock, flags);
3494

3495
	ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
3496 3497 3498 3499 3500 3501 3502 3503 3504
	if (ret) {
		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
		spin_unlock_irqrestore(&xhci->lock, flags);
		goto command_cleanup;
	}
	xhci_ring_cmd_db(xhci);
	spin_unlock_irqrestore(&xhci->lock, flags);

	/* Wait for the Reset Device command to finish */
3505
	wait_for_completion(reset_device_cmd->completion);
3506 3507 3508 3509 3510 3511 3512

	/* The Reset Device command can't fail, according to the 0.95/0.96 spec,
	 * unless we tried to reset a slot ID that wasn't enabled,
	 * or the device wasn't in the addressed or configured state.
	 */
	ret = reset_device_cmd->status;
	switch (ret) {
3513 3514 3515 3516 3517
	case COMP_CMD_ABORT:
	case COMP_CMD_STOP:
		xhci_warn(xhci, "Timeout waiting for reset device command\n");
		ret = -ETIME;
		goto command_cleanup;
3518 3519
	case COMP_EBADSLT: /* 0.95 completion code for bad slot ID */
	case COMP_CTX_STATE: /* 0.96 completion code for same thing */
3520
		xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
3521 3522
				slot_id,
				xhci_get_slot_state(xhci, virt_dev->out_ctx));
3523
		xhci_dbg(xhci, "Not freeing device rings.\n");
3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538
		/* Don't treat this as an error.  May change my mind later. */
		ret = 0;
		goto command_cleanup;
	case COMP_SUCCESS:
		xhci_dbg(xhci, "Successful reset device command.\n");
		break;
	default:
		if (xhci_is_vendor_info_code(xhci, ret))
			break;
		xhci_warn(xhci, "Unknown completion code %u for "
				"reset device command.\n", ret);
		ret = -EINVAL;
		goto command_cleanup;
	}

3539 3540 3541 3542 3543 3544 3545 3546
	/* Free up host controller endpoint resources */
	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
		spin_lock_irqsave(&xhci->lock, flags);
		/* Don't delete the default control endpoint resources */
		xhci_free_device_endpoint_resources(xhci, virt_dev, false);
		spin_unlock_irqrestore(&xhci->lock, flags);
	}

3547 3548 3549
	/* Everything but endpoint 0 is disabled, so free or cache the rings. */
	last_freed_endpoint = 1;
	for (i = 1; i < 31; ++i) {
3550 3551 3552
		struct xhci_virt_ep *ep = &virt_dev->eps[i];

		if (ep->ep_state & EP_HAS_STREAMS) {
3553 3554
			xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
					xhci_get_endpoint_address(i));
3555 3556 3557 3558 3559 3560 3561 3562 3563
			xhci_free_stream_info(xhci, ep->stream_info);
			ep->stream_info = NULL;
			ep->ep_state &= ~EP_HAS_STREAMS;
		}

		if (ep->ring) {
			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
			last_freed_endpoint = i;
		}
3564 3565 3566 3567 3568 3569 3570
		if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
			xhci_drop_ep_from_interval_table(xhci,
					&virt_dev->eps[i].bw_info,
					virt_dev->bw_table,
					udev,
					&virt_dev->eps[i],
					virt_dev->tt_info);
3571
		xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3572
	}
3573 3574 3575
	/* If necessary, update the number of active TTs on this root port */
	xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);

3576 3577 3578 3579 3580 3581 3582 3583 3584
	xhci_dbg(xhci, "Output context after successful reset device cmd:\n");
	xhci_dbg_ctx(xhci, virt_dev->out_ctx, last_freed_endpoint);
	ret = 0;

command_cleanup:
	xhci_free_command(xhci, reset_device_cmd);
	return ret;
}

3585 3586 3587 3588 3589 3590 3591 3592
/*
 * At this point, the struct usb_device is about to go away, the device has
 * disconnected, and all traffic has been stopped and the endpoints have been
 * disabled.  Free any HC data structures associated with that device.
 */
void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3593
	struct xhci_virt_device *virt_dev;
3594
	unsigned long flags;
3595
	u32 state;
3596
	int i, ret;
3597 3598 3599 3600 3601
	struct xhci_command *command;

	command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
	if (!command)
		return;
3602

3603 3604 3605 3606 3607 3608 3609
#ifndef CONFIG_USB_DEFAULT_PERSIST
	/*
	 * We called pm_runtime_get_noresume when the device was attached.
	 * Decrement the counter here to allow controller to runtime suspend
	 * if no devices remain.
	 */
	if (xhci->quirks & XHCI_RESET_ON_RESUME)
3610
		pm_runtime_put_noidle(hcd->self.controller);
3611 3612
#endif

3613
	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3614 3615 3616
	/* If the host is halted due to driver unload, we still need to free the
	 * device.
	 */
3617 3618
	if (ret <= 0 && ret != -ENODEV) {
		kfree(command);
3619
		return;
3620
	}
3621

3622 3623 3624 3625 3626 3627 3628
	virt_dev = xhci->devs[udev->slot_id];

	/* Stop any wayward timer functions (which may grab the lock) */
	for (i = 0; i < 31; ++i) {
		virt_dev->eps[i].ep_state &= ~EP_HALT_PENDING;
		del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
	}
3629 3630

	spin_lock_irqsave(&xhci->lock, flags);
3631
	/* Don't disable the slot if the host controller is dead. */
3632
	state = readl(&xhci->op_regs->status);
3633 3634
	if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
			(xhci->xhc_state & XHCI_STATE_HALTED)) {
3635 3636
		xhci_free_virt_device(xhci, udev->slot_id);
		spin_unlock_irqrestore(&xhci->lock, flags);
3637
		kfree(command);
3638 3639 3640
		return;
	}

3641 3642
	if (xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
				    udev->slot_id)) {
3643 3644 3645 3646
		spin_unlock_irqrestore(&xhci->lock, flags);
		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
		return;
	}
3647
	xhci_ring_cmd_db(xhci);
3648
	spin_unlock_irqrestore(&xhci->lock, flags);
3649

3650 3651
	/*
	 * Event command completion handler will free any data structures
3652
	 * associated with the slot.  XXX Can free sleep?
3653 3654 3655
	 */
}

3656 3657 3658 3659 3660 3661 3662 3663 3664
/*
 * Checks if we have enough host controller resources for the default control
 * endpoint.
 *
 * Must be called with xhci->lock held.
 */
static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
{
	if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
3665 3666 3667
		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
				"Not enough ep ctxs: "
				"%u active, need to add 1, limit is %u.",
3668 3669 3670 3671
				xhci->num_active_eps, xhci->limit_active_eps);
		return -ENOMEM;
	}
	xhci->num_active_eps += 1;
3672 3673
	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
			"Adding 1 ep ctx, %u now active.",
3674 3675 3676 3677 3678
			xhci->num_active_eps);
	return 0;
}


3679 3680 3681 3682 3683 3684 3685 3686
/*
 * Returns 0 if the xHC ran out of device slots, the Enable Slot command
 * timed out, or allocating memory failed.  Returns 1 on success.
 */
int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
	unsigned long flags;
3687
	int ret, slot_id;
3688 3689
	struct xhci_command *command;

3690
	command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
3691 3692
	if (!command)
		return 0;
3693

3694 3695
	/* xhci->slot_id and xhci->addr_dev are not thread-safe */
	mutex_lock(&xhci->mutex);
3696
	spin_lock_irqsave(&xhci->lock, flags);
3697
	ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
3698 3699
	if (ret) {
		spin_unlock_irqrestore(&xhci->lock, flags);
3700
		mutex_unlock(&xhci->mutex);
3701
		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3702
		xhci_free_command(xhci, command);
3703 3704
		return 0;
	}
3705
	xhci_ring_cmd_db(xhci);
3706 3707
	spin_unlock_irqrestore(&xhci->lock, flags);

3708
	wait_for_completion(command->completion);
3709 3710
	slot_id = xhci->slot_id;
	mutex_unlock(&xhci->mutex);
3711

3712
	if (!slot_id || command->status != COMP_SUCCESS) {
3713
		xhci_err(xhci, "Error while assigning device slot ID\n");
3714 3715 3716
		xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
				HCS_MAX_SLOTS(
					readl(&xhci->cap_regs->hcs_params1)));
3717
		xhci_free_command(xhci, command);
3718 3719
		return 0;
	}
3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733

	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
		spin_lock_irqsave(&xhci->lock, flags);
		ret = xhci_reserve_host_control_ep_resources(xhci);
		if (ret) {
			spin_unlock_irqrestore(&xhci->lock, flags);
			xhci_warn(xhci, "Not enough host resources, "
					"active endpoint contexts = %u\n",
					xhci->num_active_eps);
			goto disable_slot;
		}
		spin_unlock_irqrestore(&xhci->lock, flags);
	}
	/* Use GFP_NOIO, since this function can be called from
3734 3735 3736
	 * xhci_discover_or_reset_device(), which may be called as part of
	 * mass storage driver error handling.
	 */
3737
	if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {
3738
		xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
3739
		goto disable_slot;
3740
	}
3741
	udev->slot_id = slot_id;
3742 3743 3744 3745 3746 3747 3748

#ifndef CONFIG_USB_DEFAULT_PERSIST
	/*
	 * If resetting upon resume, we can't put the controller into runtime
	 * suspend if there is a device attached.
	 */
	if (xhci->quirks & XHCI_RESET_ON_RESUME)
3749
		pm_runtime_get_noresume(hcd->self.controller);
3750 3751
#endif

3752

3753
	xhci_free_command(xhci, command);
3754 3755 3756
	/* Is this a LS or FS device under a HS hub? */
	/* Hub or peripherial? */
	return 1;
3757 3758 3759 3760

disable_slot:
	/* Disable slot, if we can do it without mem alloc */
	spin_lock_irqsave(&xhci->lock, flags);
3761
	kfree(command->completion);
3762 3763 3764 3765
	command->completion = NULL;
	command->status = 0;
	if (!xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
				     udev->slot_id))
3766 3767 3768
		xhci_ring_cmd_db(xhci);
	spin_unlock_irqrestore(&xhci->lock, flags);
	return 0;
3769 3770 3771
}

/*
3772 3773
 * Issue an Address Device command and optionally send a corresponding
 * SetAddress request to the device.
3774
 */
3775 3776
static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
			     enum xhci_setup_dev setup)
3777
{
3778
	const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
3779 3780 3781 3782
	unsigned long flags;
	struct xhci_virt_device *virt_dev;
	int ret = 0;
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3783 3784
	struct xhci_slot_ctx *slot_ctx;
	struct xhci_input_control_ctx *ctrl_ctx;
3785
	u64 temp_64;
3786 3787 3788
	struct xhci_command *command = NULL;

	mutex_lock(&xhci->mutex);
3789

3790
	if (xhci->xhc_state)	/* dying, removing or halted */
3791 3792
		goto out;

3793
	if (!udev->slot_id) {
3794 3795
		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
				"Bad Slot ID %d", udev->slot_id);
3796 3797
		ret = -EINVAL;
		goto out;
3798 3799 3800 3801
	}

	virt_dev = xhci->devs[udev->slot_id];

3802 3803 3804 3805 3806 3807 3808 3809
	if (WARN_ON(!virt_dev)) {
		/*
		 * In plug/unplug torture test with an NEC controller,
		 * a zero-dereference was observed once due to virt_dev = 0.
		 * Print useful debug rather than crash if it is observed again!
		 */
		xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
			udev->slot_id);
3810 3811
		ret = -EINVAL;
		goto out;
3812 3813
	}

3814 3815 3816 3817 3818
	if (setup == SETUP_CONTEXT_ONLY) {
		slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
		if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
		    SLOT_STATE_DEFAULT) {
			xhci_dbg(xhci, "Slot already in default state\n");
3819
			goto out;
3820 3821 3822
		}
	}

3823
	command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
3824 3825 3826 3827
	if (!command) {
		ret = -ENOMEM;
		goto out;
	}
3828 3829 3830

	command->in_ctx = virt_dev->in_ctx;

3831
	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
3832
	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
3833 3834 3835
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
3836 3837
		ret = -EINVAL;
		goto out;
3838
	}
3839 3840 3841 3842 3843 3844
	/*
	 * If this is the first Set Address since device plug-in or
	 * virt_device realloaction after a resume with an xHCI power loss,
	 * then set up the slot context.
	 */
	if (!slot_ctx->dev_info)
3845
		xhci_setup_addressable_virt_dev(xhci, udev);
3846
	/* Otherwise, update the control endpoint ring enqueue pointer. */
3847 3848
	else
		xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
3849 3850 3851
	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
	ctrl_ctx->drop_flags = 0;

3852
	xhci_dbg(xhci, "Slot ID %d Input Context:\n", udev->slot_id);
3853
	xhci_dbg_ctx(xhci, virt_dev->in_ctx, 2);
3854
	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
3855
				le32_to_cpu(slot_ctx->dev_info) >> 27);
3856

3857
	spin_lock_irqsave(&xhci->lock, flags);
3858
	ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
3859
					udev->slot_id, setup);
3860 3861
	if (ret) {
		spin_unlock_irqrestore(&xhci->lock, flags);
3862 3863
		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
				"FIXME: allocate a command ring segment");
3864
		goto out;
3865
	}
3866
	xhci_ring_cmd_db(xhci);
3867 3868 3869
	spin_unlock_irqrestore(&xhci->lock, flags);

	/* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
3870 3871
	wait_for_completion(command->completion);

3872 3873 3874 3875
	/* FIXME: From section 4.3.4: "Software shall be responsible for timing
	 * the SetAddress() "recovery interval" required by USB and aborting the
	 * command on a timeout.
	 */
3876
	switch (command->status) {
3877 3878 3879 3880 3881
	case COMP_CMD_ABORT:
	case COMP_CMD_STOP:
		xhci_warn(xhci, "Timeout while waiting for setup device command\n");
		ret = -ETIME;
		break;
3882 3883
	case COMP_CTX_STATE:
	case COMP_EBADSLT:
3884 3885
		xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
			 act, udev->slot_id);
3886 3887 3888
		ret = -EINVAL;
		break;
	case COMP_TX_ERR:
3889
		dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
3890 3891
		ret = -EPROTO;
		break;
A
Alex He 已提交
3892
	case COMP_DEV_ERR:
3893 3894
		dev_warn(&udev->dev,
			 "ERROR: Incompatible device for setup %s command\n", act);
A
Alex He 已提交
3895 3896
		ret = -ENODEV;
		break;
3897
	case COMP_SUCCESS:
3898
		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3899
			       "Successful setup %s command", act);
3900 3901
		break;
	default:
3902 3903
		xhci_err(xhci,
			 "ERROR: unexpected setup %s command completion code 0x%x.\n",
3904
			 act, command->status);
3905
		xhci_dbg(xhci, "Slot ID %d Output Context:\n", udev->slot_id);
3906
		xhci_dbg_ctx(xhci, virt_dev->out_ctx, 2);
3907
		trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
3908 3909 3910
		ret = -EINVAL;
		break;
	}
3911 3912
	if (ret)
		goto out;
3913
	temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
3914 3915 3916 3917 3918 3919 3920 3921 3922 3923
	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
			"Op regs DCBAA ptr = %#016llx", temp_64);
	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
		"Slot ID %d dcbaa entry @%p = %#016llx",
		udev->slot_id,
		&xhci->dcbaa->dev_context_ptrs[udev->slot_id],
		(unsigned long long)
		le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
			"Output Context DMA address = %#08llx",
3924
			(unsigned long long)virt_dev->out_ctx->dma);
3925
	xhci_dbg(xhci, "Slot ID %d Input Context:\n", udev->slot_id);
3926
	xhci_dbg_ctx(xhci, virt_dev->in_ctx, 2);
3927
	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
3928
				le32_to_cpu(slot_ctx->dev_info) >> 27);
3929
	xhci_dbg(xhci, "Slot ID %d Output Context:\n", udev->slot_id);
3930
	xhci_dbg_ctx(xhci, virt_dev->out_ctx, 2);
3931 3932 3933 3934
	/*
	 * USB core uses address 1 for the roothubs, so we add one to the
	 * address given back to us by the HC.
	 */
3935
	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3936
	trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
3937
				le32_to_cpu(slot_ctx->dev_info) >> 27);
3938
	/* Zero the input context control for later use */
3939 3940
	ctrl_ctx->add_flags = 0;
	ctrl_ctx->drop_flags = 0;
3941

3942
	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3943 3944
		       "Internal device address = %d",
		       le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
3945 3946
out:
	mutex_unlock(&xhci->mutex);
3947 3948 3949 3950
	if (command) {
		kfree(command->completion);
		kfree(command);
	}
3951
	return ret;
3952 3953
}

3954 3955 3956 3957 3958 3959 3960 3961 3962 3963
int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
{
	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS);
}

int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
{
	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY);
}

3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976
/*
 * Transfer the port index into real index in the HW port status
 * registers. Caculate offset between the port's PORTSC register
 * and port status base. Divide the number of per port register
 * to get the real index. The raw port number bases 1.
 */
int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
	__le32 __iomem *base_addr = &xhci->op_regs->port_status_base;
	__le32 __iomem *addr;
	int raw_port;

3977
	if (hcd->speed < HCD_USB3)
3978 3979 3980 3981 3982 3983 3984 3985
		addr = xhci->usb2_ports[port1 - 1];
	else
		addr = xhci->usb3_ports[port1 - 1];

	raw_port = (addr - base_addr)/NUM_PORT_REGS + 1;
	return raw_port;
}

3986 3987 3988 3989
/*
 * Issue an Evaluate Context command to change the Maximum Exit Latency in the
 * slot context.  If that succeeds, store the new MEL in the xhci_virt_device.
 */
3990
static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
3991 3992 3993 3994 3995 3996 3997 3998 3999 4000
			struct usb_device *udev, u16 max_exit_latency)
{
	struct xhci_virt_device *virt_dev;
	struct xhci_command *command;
	struct xhci_input_control_ctx *ctrl_ctx;
	struct xhci_slot_ctx *slot_ctx;
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&xhci->lock, flags);
4001 4002 4003 4004 4005 4006 4007 4008 4009 4010

	virt_dev = xhci->devs[udev->slot_id];

	/*
	 * virt_dev might not exists yet if xHC resumed from hibernate (S4) and
	 * xHC was re-initialized. Exit latency will be set later after
	 * hub_port_finish_reset() is done and xhci->devs[] are re-allocated
	 */

	if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
4011 4012 4013 4014 4015 4016
		spin_unlock_irqrestore(&xhci->lock, flags);
		return 0;
	}

	/* Attempt to issue an Evaluate Context command to change the MEL. */
	command = xhci->lpm_command;
4017
	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
4018 4019 4020 4021 4022 4023 4024
	if (!ctrl_ctx) {
		spin_unlock_irqrestore(&xhci->lock, flags);
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		return -ENOMEM;
	}

4025 4026 4027 4028 4029 4030 4031
	xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
	spin_unlock_irqrestore(&xhci->lock, flags);

	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
	slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
	slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
4032
	slot_ctx->dev_state = 0;
4033

4034 4035
	xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
			"Set up evaluate context for LPM MEL change.");
4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052
	xhci_dbg(xhci, "Slot %u Input Context:\n", udev->slot_id);
	xhci_dbg_ctx(xhci, command->in_ctx, 0);

	/* Issue and wait for the evaluate context command. */
	ret = xhci_configure_endpoint(xhci, udev, command,
			true, true);
	xhci_dbg(xhci, "Slot %u Output Context:\n", udev->slot_id);
	xhci_dbg_ctx(xhci, virt_dev->out_ctx, 0);

	if (!ret) {
		spin_lock_irqsave(&xhci->lock, flags);
		virt_dev->current_mel = max_exit_latency;
		spin_unlock_irqrestore(&xhci->lock, flags);
	}
	return ret;
}

4053
#ifdef CONFIG_PM
A
Andiry Xu 已提交
4054 4055 4056 4057 4058 4059

/* BESL to HIRD Encoding array for USB2 LPM */
static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
	3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};

/* Calculate HIRD/BESL for USB2 PORTPMSC*/
4060 4061
static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
					struct usb_device *udev)
A
Andiry Xu 已提交
4062
{
4063 4064 4065 4066 4067 4068
	int u2del, besl, besl_host;
	int besl_device = 0;
	u32 field;

	u2del = HCS_U2_LATENCY(xhci->hcs_params3);
	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
A
Andiry Xu 已提交
4069

4070 4071 4072
	if (field & USB_BESL_SUPPORT) {
		for (besl_host = 0; besl_host < 16; besl_host++) {
			if (xhci_besl_encoding[besl_host] >= u2del)
A
Andiry Xu 已提交
4073 4074
				break;
		}
4075 4076 4077 4078 4079
		/* Use baseline BESL value as default */
		if (field & USB_BESL_BASELINE_VALID)
			besl_device = USB_GET_BESL_BASELINE(field);
		else if (field & USB_BESL_DEEP_VALID)
			besl_device = USB_GET_BESL_DEEP(field);
A
Andiry Xu 已提交
4080 4081
	} else {
		if (u2del <= 50)
4082
			besl_host = 0;
A
Andiry Xu 已提交
4083
		else
4084
			besl_host = (u2del - 51) / 75 + 1;
A
Andiry Xu 已提交
4085 4086
	}

4087 4088 4089 4090 4091
	besl = besl_host + besl_device;
	if (besl > 15)
		besl = 15;

	return besl;
A
Andiry Xu 已提交
4092 4093
}

4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104
/* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */
static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
{
	u32 field;
	int l1;
	int besld = 0;
	int hirdm = 0;

	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);

	/* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */
4105
	l1 = udev->l1_params.timeout / 256;
4106 4107 4108 4109 4110 4111 4112 4113 4114 4115

	/* device has preferred BESLD */
	if (field & USB_BESL_DEEP_VALID) {
		besld = USB_GET_BESL_DEEP(field);
		hirdm = 1;
	}

	return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
}

A
Andiry Xu 已提交
4116 4117 4118 4119 4120
int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
			struct usb_device *udev, int enable)
{
	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
	__le32 __iomem	**port_array;
4121 4122
	__le32 __iomem	*pm_addr, *hlpm_addr;
	u32		pm_val, hlpm_val, field;
A
Andiry Xu 已提交
4123 4124
	unsigned int	port_num;
	unsigned long	flags;
4125 4126
	int		hird, exit_latency;
	int		ret;
A
Andiry Xu 已提交
4127

4128
	if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||
A
Andiry Xu 已提交
4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142
			!udev->lpm_capable)
		return -EPERM;

	if (!udev->parent || udev->parent->parent ||
			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
		return -EPERM;

	if (udev->usb2_hw_lpm_capable != 1)
		return -EPERM;

	spin_lock_irqsave(&xhci->lock, flags);

	port_array = xhci->usb2_ports;
	port_num = udev->portnum - 1;
4143
	pm_addr = port_array[port_num] + PORTPMSC;
4144
	pm_val = readl(pm_addr);
4145 4146
	hlpm_addr = port_array[port_num] + PORTHLPMC;
	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
A
Andiry Xu 已提交
4147 4148

	xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
4149
			enable ? "enable" : "disable", port_num + 1);
A
Andiry Xu 已提交
4150 4151

	if (enable) {
4152 4153 4154 4155 4156 4157 4158 4159 4160 4161
		/* Host supports BESL timeout instead of HIRD */
		if (udev->usb2_hw_lpm_besl_capable) {
			/* if device doesn't have a preferred BESL value use a
			 * default one which works with mixed HIRD and BESL
			 * systems. See XHCI_DEFAULT_BESL definition in xhci.h
			 */
			if ((field & USB_BESL_SUPPORT) &&
			    (field & USB_BESL_BASELINE_VALID))
				hird = USB_GET_BESL_BASELINE(field);
			else
4162
				hird = udev->l1_params.besl;
4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183

			exit_latency = xhci_besl_encoding[hird];
			spin_unlock_irqrestore(&xhci->lock, flags);

			/* USB 3.0 code dedicate one xhci->lpm_command->in_ctx
			 * input context for link powermanagement evaluate
			 * context commands. It is protected by hcd->bandwidth
			 * mutex and is shared by all devices. We need to set
			 * the max ext latency in USB 2 BESL LPM as well, so
			 * use the same mutex and xhci_change_max_exit_latency()
			 */
			mutex_lock(hcd->bandwidth_mutex);
			ret = xhci_change_max_exit_latency(xhci, udev,
							   exit_latency);
			mutex_unlock(hcd->bandwidth_mutex);

			if (ret < 0)
				return ret;
			spin_lock_irqsave(&xhci->lock, flags);

			hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
4184
			writel(hlpm_val, hlpm_addr);
4185
			/* flush write */
4186
			readl(hlpm_addr);
4187 4188 4189 4190 4191
		} else {
			hird = xhci_calculate_hird_besl(xhci, udev);
		}

		pm_val &= ~PORT_HIRD_MASK;
4192
		pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
4193
		writel(pm_val, pm_addr);
4194
		pm_val = readl(pm_addr);
4195
		pm_val |= PORT_HLE;
4196
		writel(pm_val, pm_addr);
4197
		/* flush write */
4198
		readl(pm_addr);
A
Andiry Xu 已提交
4199
	} else {
4200
		pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
4201
		writel(pm_val, pm_addr);
4202
		/* flush write */
4203
		readl(pm_addr);
4204 4205 4206 4207 4208 4209 4210
		if (udev->usb2_hw_lpm_besl_capable) {
			spin_unlock_irqrestore(&xhci->lock, flags);
			mutex_lock(hcd->bandwidth_mutex);
			xhci_change_max_exit_latency(xhci, udev, 0);
			mutex_unlock(hcd->bandwidth_mutex);
			return 0;
		}
A
Andiry Xu 已提交
4211 4212 4213 4214 4215 4216
	}

	spin_unlock_irqrestore(&xhci->lock, flags);
	return 0;
}

4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239
/* check if a usb2 port supports a given extened capability protocol
 * only USB2 ports extended protocol capability values are cached.
 * Return 1 if capability is supported
 */
static int xhci_check_usb2_port_capability(struct xhci_hcd *xhci, int port,
					   unsigned capability)
{
	u32 port_offset, port_count;
	int i;

	for (i = 0; i < xhci->num_ext_caps; i++) {
		if (xhci->ext_caps[i] & capability) {
			/* port offsets starts at 1 */
			port_offset = XHCI_EXT_PORT_OFF(xhci->ext_caps[i]) - 1;
			port_count = XHCI_EXT_PORT_COUNT(xhci->ext_caps[i]);
			if (port >= port_offset &&
			    port < port_offset + port_count)
				return 1;
		}
	}
	return 0;
}

4240 4241 4242
int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
{
	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4243
	int		portnum = udev->portnum - 1;
4244

4245
	if (hcd->speed >= HCD_USB3 || !xhci->sw_lpm_support ||
4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262
			!udev->lpm_capable)
		return 0;

	/* we only support lpm for non-hub device connected to root hub yet */
	if (!udev->parent || udev->parent->parent ||
			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
		return 0;

	if (xhci->hw_lpm_support == 1 &&
			xhci_check_usb2_port_capability(
				xhci, portnum, XHCI_HLC)) {
		udev->usb2_hw_lpm_capable = 1;
		udev->l1_params.timeout = XHCI_L1_TIMEOUT;
		udev->l1_params.besl = XHCI_DEFAULT_BESL;
		if (xhci_check_usb2_port_capability(xhci, portnum,
					XHCI_BLC))
			udev->usb2_hw_lpm_besl_capable = 1;
4263 4264 4265 4266 4267
	}

	return 0;
}

4268 4269
/*---------------------- USB 3.0 Link PM functions ------------------------*/

4270 4271 4272 4273
/* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
static unsigned long long xhci_service_interval_to_ns(
		struct usb_endpoint_descriptor *desc)
{
O
Oliver Neukum 已提交
4274
	return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
4275 4276
}

4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301
static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
		enum usb3_link_state state)
{
	unsigned long long sel;
	unsigned long long pel;
	unsigned int max_sel_pel;
	char *state_name;

	switch (state) {
	case USB3_LPM_U1:
		/* Convert SEL and PEL stored in nanoseconds to microseconds */
		sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
		pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
		max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
		state_name = "U1";
		break;
	case USB3_LPM_U2:
		sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
		pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
		max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
		state_name = "U2";
		break;
	default:
		dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
				__func__);
S
Sarah Sharp 已提交
4302
		return USB3_LPM_DISABLED;
4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313
	}

	if (sel <= max_sel_pel && pel <= max_sel_pel)
		return USB3_LPM_DEVICE_INITIATED;

	if (sel > max_sel_pel)
		dev_dbg(&udev->dev, "Device-initiated %s disabled "
				"due to long SEL %llu ms\n",
				state_name, sel);
	else
		dev_dbg(&udev->dev, "Device-initiated %s disabled "
J
Joe Perches 已提交
4314
				"due to long PEL %llu ms\n",
4315 4316 4317 4318
				state_name, pel);
	return USB3_LPM_DISABLED;
}

4319
/* The U1 timeout should be the maximum of the following values:
4320 4321 4322 4323 4324 4325 4326
 *  - For control endpoints, U1 system exit latency (SEL) * 3
 *  - For bulk endpoints, U1 SEL * 5
 *  - For interrupt endpoints:
 *    - Notification EPs, U1 SEL * 3
 *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
 *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
 */
4327 4328
static unsigned long long xhci_calculate_intel_u1_timeout(
		struct usb_device *udev,
4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351
		struct usb_endpoint_descriptor *desc)
{
	unsigned long long timeout_ns;
	int ep_type;
	int intr_type;

	ep_type = usb_endpoint_type(desc);
	switch (ep_type) {
	case USB_ENDPOINT_XFER_CONTROL:
		timeout_ns = udev->u1_params.sel * 3;
		break;
	case USB_ENDPOINT_XFER_BULK:
		timeout_ns = udev->u1_params.sel * 5;
		break;
	case USB_ENDPOINT_XFER_INT:
		intr_type = usb_endpoint_interrupt_type(desc);
		if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
			timeout_ns = udev->u1_params.sel * 3;
			break;
		}
		/* Otherwise the calculation is the same as isoc eps */
	case USB_ENDPOINT_XFER_ISOC:
		timeout_ns = xhci_service_interval_to_ns(desc);
4352
		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
4353 4354 4355 4356 4357 4358 4359
		if (timeout_ns < udev->u1_params.sel * 2)
			timeout_ns = udev->u1_params.sel * 2;
		break;
	default:
		return 0;
	}

4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377
	return timeout_ns;
}

/* Returns the hub-encoded U1 timeout value. */
static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
		struct usb_device *udev,
		struct usb_endpoint_descriptor *desc)
{
	unsigned long long timeout_ns;

	if (xhci->quirks & XHCI_INTEL_HOST)
		timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
	else
		timeout_ns = udev->u1_params.sel;

	/* The U1 timeout is encoded in 1us intervals.
	 * Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
	 */
4378
	if (timeout_ns == USB3_LPM_DISABLED)
4379 4380 4381
		timeout_ns = 1;
	else
		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392

	/* If the necessary timeout value is bigger than what we can set in the
	 * USB 3.0 hub, we have to disable hub-initiated U1.
	 */
	if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
		return timeout_ns;
	dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
			"due to long timeout %llu ms\n", timeout_ns);
	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
}

4393
/* The U2 timeout should be the maximum of:
4394 4395 4396 4397 4398
 *  - 10 ms (to avoid the bandwidth impact on the scheduler)
 *  - largest bInterval of any active periodic endpoint (to avoid going
 *    into lower power link states between intervals).
 *  - the U2 Exit Latency of the device
 */
4399 4400
static unsigned long long xhci_calculate_intel_u2_timeout(
		struct usb_device *udev,
4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411
		struct usb_endpoint_descriptor *desc)
{
	unsigned long long timeout_ns;
	unsigned long long u2_del_ns;

	timeout_ns = 10 * 1000 * 1000;

	if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
			(xhci_service_interval_to_ns(desc) > timeout_ns))
		timeout_ns = xhci_service_interval_to_ns(desc);

4412
	u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
4413 4414 4415
	if (u2_del_ns > timeout_ns)
		timeout_ns = u2_del_ns;

4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430
	return timeout_ns;
}

/* Returns the hub-encoded U2 timeout value. */
static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
		struct usb_device *udev,
		struct usb_endpoint_descriptor *desc)
{
	unsigned long long timeout_ns;

	if (xhci->quirks & XHCI_INTEL_HOST)
		timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
	else
		timeout_ns = udev->u2_params.sel;

4431
	/* The U2 timeout is encoded in 256us intervals */
4432
	timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
4433 4434 4435 4436 4437 4438 4439 4440 4441 4442
	/* If the necessary timeout value is bigger than what we can set in the
	 * USB 3.0 hub, we have to disable hub-initiated U2.
	 */
	if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
		return timeout_ns;
	dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
			"due to long timeout %llu ms\n", timeout_ns);
	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
}

4443 4444 4445 4446 4447 4448
static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
		struct usb_device *udev,
		struct usb_endpoint_descriptor *desc,
		enum usb3_link_state state,
		u16 *timeout)
{
4449 4450 4451 4452
	if (state == USB3_LPM_U1)
		return xhci_calculate_u1_timeout(xhci, udev, desc);
	else if (state == USB3_LPM_U2)
		return xhci_calculate_u2_timeout(xhci, udev, desc);
4453

4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498
	return USB3_LPM_DISABLED;
}

static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
		struct usb_device *udev,
		struct usb_endpoint_descriptor *desc,
		enum usb3_link_state state,
		u16 *timeout)
{
	u16 alt_timeout;

	alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
		desc, state, timeout);

	/* If we found we can't enable hub-initiated LPM, or
	 * the U1 or U2 exit latency was too high to allow
	 * device-initiated LPM as well, just stop searching.
	 */
	if (alt_timeout == USB3_LPM_DISABLED ||
			alt_timeout == USB3_LPM_DEVICE_INITIATED) {
		*timeout = alt_timeout;
		return -E2BIG;
	}
	if (alt_timeout > *timeout)
		*timeout = alt_timeout;
	return 0;
}

static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
		struct usb_device *udev,
		struct usb_host_interface *alt,
		enum usb3_link_state state,
		u16 *timeout)
{
	int j;

	for (j = 0; j < alt->desc.bNumEndpoints; j++) {
		if (xhci_update_timeout_for_endpoint(xhci, udev,
					&alt->endpoint[j].desc, state, timeout))
			return -E2BIG;
		continue;
	}
	return 0;
}

4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522
static int xhci_check_intel_tier_policy(struct usb_device *udev,
		enum usb3_link_state state)
{
	struct usb_device *parent;
	unsigned int num_hubs;

	if (state == USB3_LPM_U2)
		return 0;

	/* Don't enable U1 if the device is on a 2nd tier hub or lower. */
	for (parent = udev->parent, num_hubs = 0; parent->parent;
			parent = parent->parent)
		num_hubs++;

	if (num_hubs < 2)
		return 0;

	dev_dbg(&udev->dev, "Disabling U1 link state for device"
			" below second-tier hub.\n");
	dev_dbg(&udev->dev, "Plug device into first-tier hub "
			"to decrease power consumption.\n");
	return -E2BIG;
}

4523 4524 4525 4526
static int xhci_check_tier_policy(struct xhci_hcd *xhci,
		struct usb_device *udev,
		enum usb3_link_state state)
{
4527 4528
	if (xhci->quirks & XHCI_INTEL_HOST)
		return xhci_check_intel_tier_policy(udev, state);
4529 4530
	else
		return 0;
4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570
}

/* Returns the U1 or U2 timeout that should be enabled.
 * If the tier check or timeout setting functions return with a non-zero exit
 * code, that means the timeout value has been finalized and we shouldn't look
 * at any more endpoints.
 */
static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
			struct usb_device *udev, enum usb3_link_state state)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
	struct usb_host_config *config;
	char *state_name;
	int i;
	u16 timeout = USB3_LPM_DISABLED;

	if (state == USB3_LPM_U1)
		state_name = "U1";
	else if (state == USB3_LPM_U2)
		state_name = "U2";
	else {
		dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
				state);
		return timeout;
	}

	if (xhci_check_tier_policy(xhci, udev, state) < 0)
		return timeout;

	/* Gather some information about the currently installed configuration
	 * and alternate interface settings.
	 */
	if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
			state, &timeout))
		return timeout;

	config = udev->actconfig;
	if (!config)
		return timeout;

4571
	for (i = 0; i < config->desc.bNumInterfaces; i++) {
4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691
		struct usb_driver *driver;
		struct usb_interface *intf = config->interface[i];

		if (!intf)
			continue;

		/* Check if any currently bound drivers want hub-initiated LPM
		 * disabled.
		 */
		if (intf->dev.driver) {
			driver = to_usb_driver(intf->dev.driver);
			if (driver && driver->disable_hub_initiated_lpm) {
				dev_dbg(&udev->dev, "Hub-initiated %s disabled "
						"at request of driver %s\n",
						state_name, driver->name);
				return xhci_get_timeout_no_hub_lpm(udev, state);
			}
		}

		/* Not sure how this could happen... */
		if (!intf->cur_altsetting)
			continue;

		if (xhci_update_timeout_for_interface(xhci, udev,
					intf->cur_altsetting,
					state, &timeout))
			return timeout;
	}
	return timeout;
}

static int calculate_max_exit_latency(struct usb_device *udev,
		enum usb3_link_state state_changed,
		u16 hub_encoded_timeout)
{
	unsigned long long u1_mel_us = 0;
	unsigned long long u2_mel_us = 0;
	unsigned long long mel_us = 0;
	bool disabling_u1;
	bool disabling_u2;
	bool enabling_u1;
	bool enabling_u2;

	disabling_u1 = (state_changed == USB3_LPM_U1 &&
			hub_encoded_timeout == USB3_LPM_DISABLED);
	disabling_u2 = (state_changed == USB3_LPM_U2 &&
			hub_encoded_timeout == USB3_LPM_DISABLED);

	enabling_u1 = (state_changed == USB3_LPM_U1 &&
			hub_encoded_timeout != USB3_LPM_DISABLED);
	enabling_u2 = (state_changed == USB3_LPM_U2 &&
			hub_encoded_timeout != USB3_LPM_DISABLED);

	/* If U1 was already enabled and we're not disabling it,
	 * or we're going to enable U1, account for the U1 max exit latency.
	 */
	if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
			enabling_u1)
		u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
	if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
			enabling_u2)
		u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);

	if (u1_mel_us > u2_mel_us)
		mel_us = u1_mel_us;
	else
		mel_us = u2_mel_us;
	/* xHCI host controller max exit latency field is only 16 bits wide. */
	if (mel_us > MAX_EXIT) {
		dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
				"is too big.\n", mel_us);
		return -E2BIG;
	}
	return mel_us;
}

/* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
			struct usb_device *udev, enum usb3_link_state state)
{
	struct xhci_hcd	*xhci;
	u16 hub_encoded_timeout;
	int mel;
	int ret;

	xhci = hcd_to_xhci(hcd);
	/* The LPM timeout values are pretty host-controller specific, so don't
	 * enable hub-initiated timeouts unless the vendor has provided
	 * information about their timeout algorithm.
	 */
	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
			!xhci->devs[udev->slot_id])
		return USB3_LPM_DISABLED;

	hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
	mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
	if (mel < 0) {
		/* Max Exit Latency is too big, disable LPM. */
		hub_encoded_timeout = USB3_LPM_DISABLED;
		mel = 0;
	}

	ret = xhci_change_max_exit_latency(xhci, udev, mel);
	if (ret)
		return ret;
	return hub_encoded_timeout;
}

int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
			struct usb_device *udev, enum usb3_link_state state)
{
	struct xhci_hcd	*xhci;
	u16 mel;

	xhci = hcd_to_xhci(hcd);
	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
			!xhci->devs[udev->slot_id])
		return 0;

	mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
4692
	return xhci_change_max_exit_latency(xhci, udev, mel);
4693
}
4694
#else /* CONFIG_PM */
A
Andiry Xu 已提交
4695

4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706
int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
				struct usb_device *udev, int enable)
{
	return 0;
}

int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
{
	return 0;
}

4707 4708
int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
			struct usb_device *udev, enum usb3_link_state state)
A
Andiry Xu 已提交
4709
{
4710
	return USB3_LPM_DISABLED;
A
Andiry Xu 已提交
4711 4712
}

4713 4714
int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
			struct usb_device *udev, enum usb3_link_state state)
A
Andiry Xu 已提交
4715 4716 4717
{
	return 0;
}
4718
#endif	/* CONFIG_PM */
A
Andiry Xu 已提交
4719

4720
/*-------------------------------------------------------------------------*/
A
Andiry Xu 已提交
4721

S
Sarah Sharp 已提交
4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745
/* Once a hub descriptor is fetched for a device, we need to update the xHC's
 * internal data structures for the device.
 */
int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
			struct usb_tt *tt, gfp_t mem_flags)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
	struct xhci_virt_device *vdev;
	struct xhci_command *config_cmd;
	struct xhci_input_control_ctx *ctrl_ctx;
	struct xhci_slot_ctx *slot_ctx;
	unsigned long flags;
	unsigned think_time;
	int ret;

	/* Ignore root hubs */
	if (!hdev->parent)
		return 0;

	vdev = xhci->devs[hdev->slot_id];
	if (!vdev) {
		xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
		return -EINVAL;
	}
4746
	config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
S
Sarah Sharp 已提交
4747 4748 4749 4750
	if (!config_cmd) {
		xhci_dbg(xhci, "Could not allocate xHCI command structure.\n");
		return -ENOMEM;
	}
4751
	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
4752 4753 4754 4755 4756 4757
	if (!ctrl_ctx) {
		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
				__func__);
		xhci_free_command(xhci, config_cmd);
		return -ENOMEM;
	}
S
Sarah Sharp 已提交
4758 4759

	spin_lock_irqsave(&xhci->lock, flags);
4760 4761 4762 4763 4764 4765 4766 4767
	if (hdev->speed == USB_SPEED_HIGH &&
			xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
		xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
		xhci_free_command(xhci, config_cmd);
		spin_unlock_irqrestore(&xhci->lock, flags);
		return -ENOMEM;
	}

S
Sarah Sharp 已提交
4768
	xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
M
Matt Evans 已提交
4769
	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
S
Sarah Sharp 已提交
4770
	slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
M
Matt Evans 已提交
4771
	slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
4772 4773 4774 4775 4776
	/*
	 * refer to section 6.2.2: MTT should be 0 for full speed hub,
	 * but it may be already set to 1 when setup an xHCI virtual
	 * device, so clear it anyway.
	 */
S
Sarah Sharp 已提交
4777
	if (tt->multi)
M
Matt Evans 已提交
4778
		slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
4779 4780 4781
	else if (hdev->speed == USB_SPEED_FULL)
		slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);

S
Sarah Sharp 已提交
4782 4783 4784 4785
	if (xhci->hci_version > 0x95) {
		xhci_dbg(xhci, "xHCI version %x needs hub "
				"TT think time and number of ports\n",
				(unsigned int) xhci->hci_version);
M
Matt Evans 已提交
4786
		slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
S
Sarah Sharp 已提交
4787 4788 4789
		/* Set TT think time - convert from ns to FS bit times.
		 * 0 = 8 FS bit times, 1 = 16 FS bit times,
		 * 2 = 24 FS bit times, 3 = 32 FS bit times.
A
Andiry Xu 已提交
4790 4791 4792
		 *
		 * xHCI 1.0: this field shall be 0 if the device is not a
		 * High-spped hub.
S
Sarah Sharp 已提交
4793 4794 4795 4796
		 */
		think_time = tt->think_time;
		if (think_time != 0)
			think_time = (think_time / 666) - 1;
A
Andiry Xu 已提交
4797 4798 4799
		if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
			slot_ctx->tt_info |=
				cpu_to_le32(TT_THINK_TIME(think_time));
S
Sarah Sharp 已提交
4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830
	} else {
		xhci_dbg(xhci, "xHCI version %x doesn't need hub "
				"TT think time or number of ports\n",
				(unsigned int) xhci->hci_version);
	}
	slot_ctx->dev_state = 0;
	spin_unlock_irqrestore(&xhci->lock, flags);

	xhci_dbg(xhci, "Set up %s for hub device.\n",
			(xhci->hci_version > 0x95) ?
			"configure endpoint" : "evaluate context");
	xhci_dbg(xhci, "Slot %u Input Context:\n", hdev->slot_id);
	xhci_dbg_ctx(xhci, config_cmd->in_ctx, 0);

	/* Issue and wait for the configure endpoint or
	 * evaluate context command.
	 */
	if (xhci->hci_version > 0x95)
		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
				false, false);
	else
		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
				true, false);

	xhci_dbg(xhci, "Slot %u Output Context:\n", hdev->slot_id);
	xhci_dbg_ctx(xhci, vdev->out_ctx, 0);

	xhci_free_command(xhci, config_cmd);
	return ret;
}

4831 4832 4833 4834
int xhci_get_frame(struct usb_hcd *hcd)
{
	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
	/* EHCI mods by the periodic size.  Why? */
4835
	return readl(&xhci->run_regs->microframe_index) >> 3;
4836 4837
}

4838 4839 4840 4841 4842 4843
int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
{
	struct xhci_hcd		*xhci;
	struct device		*dev = hcd->self.controller;
	int			retval;

4844 4845
	/* Accept arbitrarily long scatter-gather lists */
	hcd->self.sg_tablesize = ~0;
M
Ming Lei 已提交
4846

4847 4848 4849
	/* support to build packet from discontinuous buffers */
	hcd->self.no_sg_constraint = 1;

4850 4851
	/* XHCI controllers don't stop the ep queue on short packets :| */
	hcd->self.no_stop_on_short = 1;
4852

4853 4854
	xhci = hcd_to_xhci(hcd);

4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868
	if (usb_hcd_is_primary_hcd(hcd)) {
		xhci->main_hcd = hcd;
		/* Mark the first roothub as being USB 2.0.
		 * The xHCI driver will register the USB 3.0 roothub.
		 */
		hcd->speed = HCD_USB2;
		hcd->self.root_hub->speed = USB_SPEED_HIGH;
		/*
		 * USB 2.0 roothub under xHCI has an integrated TT,
		 * (rate matching hub) as opposed to having an OHCI/UHCI
		 * companion controller.
		 */
		hcd->has_tt = 1;
	} else {
4869 4870 4871
		if (xhci->sbrn == 0x31) {
			xhci_info(xhci, "Host supports USB 3.1 Enhanced SuperSpeed\n");
			hcd->speed = HCD_USB31;
4872
			hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
4873
		}
4874 4875 4876 4877 4878 4879
		/* xHCI private pointer was set in xhci_pci_probe for the second
		 * registered roothub.
		 */
		return 0;
	}

4880
	mutex_init(&xhci->mutex);
4881 4882
	xhci->cap_regs = hcd->regs;
	xhci->op_regs = hcd->regs +
4883
		HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
4884
	xhci->run_regs = hcd->regs +
4885
		(readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
4886
	/* Cache read-only capability registers */
4887 4888 4889 4890
	xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
	xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
	xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
	xhci->hcc_params = readl(&xhci->cap_regs->hc_capbase);
4891
	xhci->hci_version = HC_VERSION(xhci->hcc_params);
4892
	xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
4893 4894
	if (xhci->hci_version > 0x100)
		xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
4895 4896
	xhci_print_registers(xhci);

4897
	xhci->quirks |= quirks;
T
Takashi Iwai 已提交
4898

4899 4900
	get_quirks(dev, xhci);

4901 4902 4903 4904 4905 4906 4907
	/* In xhci controllers which follow xhci 1.0 spec gives a spurious
	 * success event after a short transfer. This quirk will ignore such
	 * spurious event.
	 */
	if (xhci->hci_version > 0x96)
		xhci->quirks |= XHCI_SPURIOUS_SUCCESS;

4908 4909 4910
	/* Make sure the HC is halted. */
	retval = xhci_halt(xhci);
	if (retval)
4911
		return retval;
4912 4913 4914 4915 4916

	xhci_dbg(xhci, "Resetting HCD\n");
	/* Reset the internal HC memory state and registers. */
	retval = xhci_reset(xhci);
	if (retval)
4917
		return retval;
4918 4919
	xhci_dbg(xhci, "Reset complete\n");

4920 4921 4922 4923 4924 4925 4926 4927 4928 4929
	/*
	 * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0)
	 * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit
	 * address memory pointers actually. So, this driver clears the AC64
	 * bit of xhci->hcc_params to call dma_set_coherent_mask(dev,
	 * DMA_BIT_MASK(32)) in this xhci_gen_setup().
	 */
	if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
		xhci->hcc_params &= ~BIT(0);

4930 4931 4932 4933
	/* Set dma_mask and coherent_dma_mask to 64-bits,
	 * if xHC supports 64-bit addressing */
	if (HCC_64BIT_ADDR(xhci->hcc_params) &&
			!dma_set_mask(dev, DMA_BIT_MASK(64))) {
4934
		xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
4935
		dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
4936 4937 4938 4939 4940 4941 4942 4943 4944 4945
	} else {
		/*
		 * This is to avoid error in cases where a 32-bit USB
		 * controller is used on a 64-bit capable system.
		 */
		retval = dma_set_mask(dev, DMA_BIT_MASK(32));
		if (retval)
			return retval;
		xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");
		dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
4946 4947 4948 4949 4950 4951
	}

	xhci_dbg(xhci, "Calling HCD init\n");
	/* Initialize HCD and host controller data structures. */
	retval = xhci_init(hcd);
	if (retval)
4952
		return retval;
4953
	xhci_dbg(xhci, "Called HCD init\n");
4954 4955 4956 4957

	xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%08x\n",
		  xhci->hcc_params, xhci->hci_version, xhci->quirks);

4958 4959
	return 0;
}
4960
EXPORT_SYMBOL_GPL(xhci_gen_setup);
4961

4962 4963 4964
static const struct hc_driver xhci_hc_driver = {
	.description =		"xhci-hcd",
	.product_desc =		"xHCI Host Controller",
4965
	.hcd_priv_size =	sizeof(struct xhci_hcd),
4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022

	/*
	 * generic hardware linkage
	 */
	.irq =			xhci_irq,
	.flags =		HCD_MEMORY | HCD_USB3 | HCD_SHARED,

	/*
	 * basic lifecycle operations
	 */
	.reset =		NULL, /* set in xhci_init_driver() */
	.start =		xhci_run,
	.stop =			xhci_stop,
	.shutdown =		xhci_shutdown,

	/*
	 * managing i/o requests and associated device resources
	 */
	.urb_enqueue =		xhci_urb_enqueue,
	.urb_dequeue =		xhci_urb_dequeue,
	.alloc_dev =		xhci_alloc_dev,
	.free_dev =		xhci_free_dev,
	.alloc_streams =	xhci_alloc_streams,
	.free_streams =		xhci_free_streams,
	.add_endpoint =		xhci_add_endpoint,
	.drop_endpoint =	xhci_drop_endpoint,
	.endpoint_reset =	xhci_endpoint_reset,
	.check_bandwidth =	xhci_check_bandwidth,
	.reset_bandwidth =	xhci_reset_bandwidth,
	.address_device =	xhci_address_device,
	.enable_device =	xhci_enable_device,
	.update_hub_device =	xhci_update_hub_device,
	.reset_device =		xhci_discover_or_reset_device,

	/*
	 * scheduling support
	 */
	.get_frame_number =	xhci_get_frame,

	/*
	 * root hub support
	 */
	.hub_control =		xhci_hub_control,
	.hub_status_data =	xhci_hub_status_data,
	.bus_suspend =		xhci_bus_suspend,
	.bus_resume =		xhci_bus_resume,

	/*
	 * call back when device connected and addressed
	 */
	.update_device =        xhci_update_device,
	.set_usb2_hw_lpm =	xhci_set_usb2_hardware_lpm,
	.enable_usb3_lpm_timeout =	xhci_enable_usb3_lpm_timeout,
	.disable_usb3_lpm_timeout =	xhci_disable_usb3_lpm_timeout,
	.find_raw_port_number =	xhci_find_raw_port_number,
};

5023 5024
void xhci_init_driver(struct hc_driver *drv,
		      const struct xhci_driver_overrides *over)
5025
{
5026 5027 5028
	BUG_ON(!over);

	/* Copy the generic table to drv then apply the overrides */
5029
	*drv = xhci_hc_driver;
5030 5031 5032 5033 5034 5035 5036 5037

	if (over) {
		drv->hcd_priv_size += over->extra_priv_size;
		if (over->reset)
			drv->reset = over->reset;
		if (over->start)
			drv->start = over->start;
	}
5038 5039 5040
}
EXPORT_SYMBOL_GPL(xhci_init_driver);

5041 5042 5043 5044 5045 5046
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_LICENSE("GPL");

static int __init xhci_hcd_init(void)
{
5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059
	/*
	 * Check the compiler generated sizes of structures that must be laid
	 * out in specific ways for hardware access.
	 */
	BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
	BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
	BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
	/* xhci_device_control has eight fields, and also
	 * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
	 */
	BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
	BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
	BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
5060
	BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8);
5061 5062 5063
	BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
	/* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
	BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
5064 5065 5066 5067

	if (usb_disabled())
		return -ENODEV;

5068 5069
	return 0;
}
5070 5071 5072 5073 5074 5075 5076

/*
 * If an init function is provided, an exit function must also be provided
 * to allow module unload.
 */
static void __exit xhci_hcd_fini(void) { }

5077
module_init(xhci_hcd_init);
5078
module_exit(xhci_hcd_fini);