gadget.c 82.6 KB
Newer Older
F
Felipe Balbi 已提交
1
/*
2 3 4 5 6 7 8
 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
 *
 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
 *
 * Authors: Felipe Balbi <balbi@ti.com>,
 *	    Sebastian Andrzej Siewior <bigeasy@linutronix.de>
 *
F
Felipe Balbi 已提交
9 10 11
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2  of
 * the License as published by the Free Software Foundation.
12
 *
F
Felipe Balbi 已提交
13 14 15 16
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
 */

#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/list.h>
#include <linux/dma-mapping.h>

#include <linux/usb/ch9.h>
#include <linux/usb/gadget.h>

33
#include "debug.h"
34 35 36 37
#include "core.h"
#include "gadget.h"
#include "io.h"

38
/**
F
Felipe Balbi 已提交
39
 * dwc3_gadget_set_test_mode - enables usb2 test modes
40 41 42
 * @dwc: pointer to our context structure
 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
 *
F
Felipe Balbi 已提交
43 44
 * Caller should take care of locking. This function will return 0 on
 * success or -EINVAL if wrong Test Selector is passed.
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
 */
int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
{
	u32		reg;

	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_TSTCTRL_MASK;

	switch (mode) {
	case TEST_J:
	case TEST_K:
	case TEST_SE0_NAK:
	case TEST_PACKET:
	case TEST_FORCE_EN:
		reg |= mode << 1;
		break;
	default:
		return -EINVAL;
	}

	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	return 0;
}

70
/**
F
Felipe Balbi 已提交
71
 * dwc3_gadget_get_link_state - gets current state of usb link
72 73 74 75 76 77 78 79 80 81 82 83 84 85
 * @dwc: pointer to our context structure
 *
 * Caller should take care of locking. This function will
 * return the link state on success (>= 0) or -ETIMEDOUT.
 */
int dwc3_gadget_get_link_state(struct dwc3 *dwc)
{
	u32		reg;

	reg = dwc3_readl(dwc->regs, DWC3_DSTS);

	return DWC3_DSTS_USBLNKST(reg);
}

86
/**
F
Felipe Balbi 已提交
87
 * dwc3_gadget_set_link_state - sets usb link to a particular state
88 89 90 91
 * @dwc: pointer to our context structure
 * @state: the state to put link into
 *
 * Caller should take care of locking. This function will
92
 * return 0 on success or -ETIMEDOUT.
93 94 95
 */
int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
{
96
	int		retries = 10000;
97 98
	u32		reg;

99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
	/*
	 * Wait until device controller is ready. Only applies to 1.94a and
	 * later RTL.
	 */
	if (dwc->revision >= DWC3_REVISION_194A) {
		while (--retries) {
			reg = dwc3_readl(dwc->regs, DWC3_DSTS);
			if (reg & DWC3_DSTS_DCNRD)
				udelay(5);
			else
				break;
		}

		if (retries <= 0)
			return -ETIMEDOUT;
	}

116 117 118 119 120 121 122
	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;

	/* set requested state */
	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

123 124 125 126 127 128 129
	/*
	 * The following code is racy when called from dwc3_gadget_wakeup,
	 * and is not needed, at least on newer versions
	 */
	if (dwc->revision >= DWC3_REVISION_194A)
		return 0;

130
	/* wait for a change in DSTS */
131
	retries = 10000;
132 133 134 135 136 137
	while (--retries) {
		reg = dwc3_readl(dwc->regs, DWC3_DSTS);

		if (DWC3_DSTS_USBLNKST(reg) == state)
			return 0;

138
		udelay(5);
139 140 141 142 143
	}

	return -ETIMEDOUT;
}

144
/**
F
Felipe Balbi 已提交
145 146
 * dwc3_ep_inc_trb - increment a trb index.
 * @index: Pointer to the TRB index to increment.
147 148 149 150 151 152
 *
 * The index should never point to the link TRB. After incrementing,
 * if it is point to the link TRB, wrap around to the beginning. The
 * link TRB is always at the last TRB entry.
 */
static void dwc3_ep_inc_trb(u8 *index)
153
{
154 155 156
	(*index)++;
	if (*index == (DWC3_TRB_NUM - 1))
		*index = 0;
157
}
158

F
Felipe Balbi 已提交
159 160 161 162
/**
 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 * @dep: The endpoint whose enqueue pointer we're incrementing
 */
163
static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
164
{
165
	dwc3_ep_inc_trb(&dep->trb_enqueue);
166
}
167

F
Felipe Balbi 已提交
168 169 170 171
/**
 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 * @dep: The endpoint whose enqueue pointer we're incrementing
 */
172
static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
173
{
174
	dwc3_ep_inc_trb(&dep->trb_dequeue);
175 176
}

F
Felipe Balbi 已提交
177 178 179 180 181 182 183 184 185 186
/**
 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 * @dep: The endpoint to whom the request belongs to
 * @req: The request we're giving back
 * @status: completion code for the request
 *
 * Must be called with controller's lock held and interrupts disabled. This
 * function will unmap @req and call its ->complete() callback to notify upper
 * layers that it has completed.
 */
187 188 189 190 191
void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
		int status)
{
	struct dwc3			*dwc = dep->dwc;

192
	req->started = false;
193
	list_del(&req->list);
194
	req->remaining = 0;
195 196 197 198

	if (req->request.status == -EINPROGRESS)
		req->request.status = status;

199 200 201 202 203
	if (req->trb)
		usb_gadget_unmap_request_by_dev(dwc->sysdev,
						&req->request, req->direction);

	req->trb = NULL;
204

205
	trace_dwc3_gadget_giveback(req);
206 207

	spin_unlock(&dwc->lock);
208
	usb_gadget_giveback_request(&dep->endpoint, &req->request);
209
	spin_lock(&dwc->lock);
F
Felipe Balbi 已提交
210 211 212

	if (dep->number > 1)
		pm_runtime_put(dwc->dev);
213 214
}

F
Felipe Balbi 已提交
215 216 217 218 219 220 221 222 223
/**
 * dwc3_send_gadget_generic_command - issue a generic command for the controller
 * @dwc: pointer to the controller context
 * @cmd: the command to be issued
 * @param: command parameter
 *
 * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 * and wait for its completion.
 */
224
int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
225 226
{
	u32		timeout = 500;
227
	int		status = 0;
228
	int		ret = 0;
229 230 231 232 233 234 235 236
	u32		reg;

	dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
	dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);

	do {
		reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
		if (!(reg & DWC3_DGCMD_CMDACT)) {
237 238
			status = DWC3_DGCMD_STATUS(reg);
			if (status)
239 240
				ret = -EINVAL;
			break;
241
		}
J
Janusz Dziedzic 已提交
242
	} while (--timeout);
243 244 245

	if (!timeout) {
		ret = -ETIMEDOUT;
246
		status = -ETIMEDOUT;
247 248
	}

249 250
	trace_dwc3_gadget_generic_cmd(cmd, param, status);

251
	return ret;
252 253
}

254 255
static int __dwc3_gadget_wakeup(struct dwc3 *dwc);

F
Felipe Balbi 已提交
256 257 258 259 260 261 262 263 264
/**
 * dwc3_send_gadget_ep_cmd - issue an endpoint command
 * @dep: the endpoint to which the command is going to be issued
 * @cmd: the command to be issued
 * @params: parameters to the command
 *
 * Caller should handle locking. This function will issue @cmd with given
 * @params to @dep and wait for its completion.
 */
265 266
int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
		struct dwc3_gadget_ep_cmd_params *params)
267
{
268
	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
269
	struct dwc3		*dwc = dep->dwc;
270
	u32			timeout = 500;
271 272
	u32			reg;

273
	int			cmd_status = 0;
274
	int			susphy = false;
275
	int			ret = -EINVAL;
276

277 278 279 280 281 282 283 284
	/*
	 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
	 * we're issuing an endpoint command, we must check if
	 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
	 *
	 * We will also set SUSPHY bit to what it was before returning as stated
	 * by the same section on Synopsys databook.
	 */
285 286 287 288 289 290 291
	if (dwc->gadget.speed <= USB_SPEED_HIGH) {
		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
		if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
			susphy = true;
			reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
			dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
		}
292 293
	}

294
	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
295 296 297 298 299 300 301 302 303 304 305 306 307
		int		needs_wakeup;

		needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
				dwc->link_state == DWC3_LINK_STATE_U2 ||
				dwc->link_state == DWC3_LINK_STATE_U3);

		if (unlikely(needs_wakeup)) {
			ret = __dwc3_gadget_wakeup(dwc);
			dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
					ret);
		}
	}

308 309 310
	dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
	dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
	dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
311

312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
	/*
	 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
	 * not relying on XferNotReady, we can make use of a special "No
	 * Response Update Transfer" command where we should clear both CmdAct
	 * and CmdIOC bits.
	 *
	 * With this, we don't need to wait for command completion and can
	 * straight away issue further commands to the endpoint.
	 *
	 * NOTICE: We're making an assumption that control endpoints will never
	 * make use of Update Transfer command. This is a safe assumption
	 * because we can never have more than one request at a time with
	 * Control Endpoints. If anybody changes that assumption, this chunk
	 * needs to be updated accordingly.
	 */
	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
			!usb_endpoint_xfer_isoc(desc))
		cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
	else
		cmd |= DWC3_DEPCMD_CMDACT;

	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
334
	do {
335
		reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
336
		if (!(reg & DWC3_DEPCMD_CMDACT)) {
337
			cmd_status = DWC3_DEPCMD_STATUS(reg);
338 339 340 341 342 343 344

			switch (cmd_status) {
			case 0:
				ret = 0;
				break;
			case DEPEVT_TRANSFER_NO_RESOURCE:
				ret = -EINVAL;
345
				break;
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
			case DEPEVT_TRANSFER_BUS_EXPIRY:
				/*
				 * SW issues START TRANSFER command to
				 * isochronous ep with future frame interval. If
				 * future interval time has already passed when
				 * core receives the command, it will respond
				 * with an error status of 'Bus Expiry'.
				 *
				 * Instead of always returning -EINVAL, let's
				 * give a hint to the gadget driver that this is
				 * the case by returning -EAGAIN.
				 */
				ret = -EAGAIN;
				break;
			default:
				dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
			}

364
			break;
365
		}
366
	} while (--timeout);
367

368 369
	if (timeout == 0) {
		ret = -ETIMEDOUT;
370
		cmd_status = -ETIMEDOUT;
371
	}
372

373 374
	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);

375 376 377 378 379 380 381 382 383 384 385 386 387 388
	if (ret == 0) {
		switch (DWC3_DEPCMD_CMD(cmd)) {
		case DWC3_DEPCMD_STARTTRANSFER:
			dep->flags |= DWC3_EP_TRANSFER_STARTED;
			break;
		case DWC3_DEPCMD_ENDTRANSFER:
			dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
			break;
		default:
			/* nothing */
			break;
		}
	}

389 390 391 392 393 394
	if (unlikely(susphy)) {
		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
		reg |= DWC3_GUSB2PHYCFG_SUSPHY;
		dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
	}

395
	return ret;
396 397
}

398 399 400 401 402 403 404 405 406 407 408 409 410 411
static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
{
	struct dwc3 *dwc = dep->dwc;
	struct dwc3_gadget_ep_cmd_params params;
	u32 cmd = DWC3_DEPCMD_CLEARSTALL;

	/*
	 * As of core revision 2.60a the recommended programming model
	 * is to set the ClearPendIN bit when issuing a Clear Stall EP
	 * command for IN endpoints. This is to prevent an issue where
	 * some (non-compliant) hosts may not send ACK TPs for pending
	 * IN transfers due to a mishandled error condition. Synopsys
	 * STAR 9000614252.
	 */
412 413
	if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
	    (dwc->gadget.speed >= USB_SPEED_SUPER))
414 415 416 417
		cmd |= DWC3_DEPCMD_CLEARPENDIN;

	memset(&params, 0, sizeof(params));

418
	return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
419 420
}

421
static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
422
		struct dwc3_trb *trb)
423
{
424
	u32		offset = (char *) trb - (char *) dep->trb_pool;
425 426 427 428 429 430 431 432 433 434 435

	return dep->trb_pool_dma + offset;
}

static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
{
	struct dwc3		*dwc = dep->dwc;

	if (dep->trb_pool)
		return 0;

436
	dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
			sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
			&dep->trb_pool_dma, GFP_KERNEL);
	if (!dep->trb_pool) {
		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
				dep->name);
		return -ENOMEM;
	}

	return 0;
}

static void dwc3_free_trb_pool(struct dwc3_ep *dep)
{
	struct dwc3		*dwc = dep->dwc;

452
	dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
453 454 455 456 457 458
			dep->trb_pool, dep->trb_pool_dma);

	dep->trb_pool = NULL;
	dep->trb_pool_dma = 0;
}

459 460 461
static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);

/**
F
Felipe Balbi 已提交
462
 * dwc3_gadget_start_config - configure ep resources
463 464 465
 * @dwc: pointer to our controller context structure
 * @dep: endpoint that is being enabled
 *
F
Felipe Balbi 已提交
466 467
 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 * completion, it will set Transfer Resource for all available endpoints.
468
 *
F
Felipe Balbi 已提交
469 470 471 472 473 474
 * The assignment of transfer resources cannot perfectly follow the data book
 * due to the fact that the controller driver does not have all knowledge of the
 * configuration in advance. It is given this information piecemeal by the
 * composite gadget framework after every SET_CONFIGURATION and
 * SET_INTERFACE. Trying to follow the databook programming model in this
 * scenario can cause errors. For two reasons:
475
 *
F
Felipe Balbi 已提交
476 477 478 479 480
 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 * incorrect in the scenario of multiple interfaces.
 *
 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
481 482 483 484
 * endpoint on alt setting (8.1.6).
 *
 * The following simplified method is used instead:
 *
F
Felipe Balbi 已提交
485 486 487 488
 * All hardware endpoints can be assigned a transfer resource and this setting
 * will stay persistent until either a core reset or hibernation. So whenever we
 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
489 490
 * guaranteed that there are as many transfer resources as endpoints.
 *
F
Felipe Balbi 已提交
491 492 493
 * This function is called for each endpoint when it is being enabled but is
 * triggered only when called for EP0-out, which always happens first, and which
 * should only happen in one of the above conditions.
494
 */
495 496 497 498
static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
{
	struct dwc3_gadget_ep_cmd_params params;
	u32			cmd;
499 500 501 502 503
	int			i;
	int			ret;

	if (dep->number)
		return 0;
504 505

	memset(&params, 0x00, sizeof(params));
506
	cmd = DWC3_DEPCMD_DEPSTARTCFG;
507

508
	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
509 510 511 512 513
	if (ret)
		return ret;

	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
		struct dwc3_ep *dep = dwc->eps[i];
514

515 516 517 518 519 520
		if (!dep)
			continue;

		ret = dwc3_gadget_set_xfer_resource(dwc, dep);
		if (ret)
			return ret;
521 522 523 524 525 526
	}

	return 0;
}

static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
527
		bool modify, bool restore)
528
{
529 530
	const struct usb_ss_ep_comp_descriptor *comp_desc;
	const struct usb_endpoint_descriptor *desc;
531 532
	struct dwc3_gadget_ep_cmd_params params;

533 534 535 536
	if (dev_WARN_ONCE(dwc->dev, modify && restore,
					"Can't modify and restore\n"))
		return -EINVAL;

537 538 539
	comp_desc = dep->endpoint.comp_desc;
	desc = dep->endpoint.desc;

540 541
	memset(&params, 0x00, sizeof(params));

542
	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
543 544 545
		| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));

	/* Burst size is only needed in SuperSpeed mode */
546
	if (dwc->gadget.speed >= USB_SPEED_SUPER) {
547 548
		u32 burst = dep->endpoint.maxburst;
		params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
549
	}
550

551 552 553
	if (modify) {
		params.param0 |= DWC3_DEPCFG_ACTION_MODIFY;
	} else if (restore) {
554 555
		params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
		params.param2 |= dep->saved_state;
556 557
	} else {
		params.param0 |= DWC3_DEPCFG_ACTION_INIT;
558 559
	}

560 561
	if (usb_endpoint_xfer_control(desc))
		params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
562 563 564

	if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
		params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
565

566
	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
567 568
		params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
			| DWC3_DEPCFG_STREAM_EVENT_EN;
569 570 571
		dep->stream_capable = true;
	}

572
	if (!usb_endpoint_xfer_control(desc))
573
		params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
574 575 576 577 578 579 580

	/*
	 * We are doing 1:1 mapping for endpoints, meaning
	 * Physical Endpoints 2 maps to Logical Endpoint 2 and
	 * so on. We consider the direction bit as part of the physical
	 * endpoint number. So USB endpoint 0x81 is 0x03.
	 */
581
	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
582 583 584 585 586 587

	/*
	 * We must use the lower 16 TX FIFOs even though
	 * HW might have more
	 */
	if (dep->direction)
588
		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
589 590

	if (desc->bInterval) {
591
		params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
592 593 594
		dep->interval = 1 << (desc->bInterval - 1);
	}

595
	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
596 597 598 599 600 601 602 603
}

static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
{
	struct dwc3_gadget_ep_cmd_params params;

	memset(&params, 0x00, sizeof(params));

604
	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
605

606 607
	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
			&params);
608 609 610
}

/**
F
Felipe Balbi 已提交
611
 * __dwc3_gadget_ep_enable - initializes a hw endpoint
612
 * @dep: endpoint to be initialized
F
Felipe Balbi 已提交
613 614
 * @modify: if true, modify existing endpoint configuration
 * @restore: if true, restore endpoint configuration from scratch buffer
615
 *
F
Felipe Balbi 已提交
616 617
 * Caller should take care of locking. Execute all necessary commands to
 * initialize a HW endpoint so it can be used by a gadget driver.
618 619
 */
static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
620
		bool modify, bool restore)
621
{
622
	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
623
	struct dwc3		*dwc = dep->dwc;
624

625
	u32			reg;
626
	int			ret;
627 628 629 630 631 632 633

	if (!(dep->flags & DWC3_EP_ENABLED)) {
		ret = dwc3_gadget_start_config(dwc, dep);
		if (ret)
			return ret;
	}

634
	ret = dwc3_gadget_set_ep_config(dwc, dep, modify, restore);
635 636 637 638
	if (ret)
		return ret;

	if (!(dep->flags & DWC3_EP_ENABLED)) {
639 640
		struct dwc3_trb	*trb_st_hw;
		struct dwc3_trb	*trb_link;
641 642 643

		dep->type = usb_endpoint_type(desc);
		dep->flags |= DWC3_EP_ENABLED;
644
		dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
645 646 647 648 649

		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
		reg |= DWC3_DALEPENA_EP(dep->number);
		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);

650 651
		init_waitqueue_head(&dep->wait_end_transfer);

652
		if (usb_endpoint_xfer_control(desc))
653
			goto out;
654

655 656 657 658 659 660
		/* Initialize the TRB ring */
		dep->trb_dequeue = 0;
		dep->trb_enqueue = 0;
		memset(dep->trb_pool, 0,
		       sizeof(struct dwc3_trb) * DWC3_TRB_NUM);

661
		/* Link TRB. The HWO bit is never reset */
662 663
		trb_st_hw = &dep->trb_pool[0];

664 665 666 667 668
		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
669 670
	}

671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
	/*
	 * Issue StartTransfer here with no-op TRB so we can always rely on No
	 * Response Update Transfer command.
	 */
	if (usb_endpoint_xfer_bulk(desc)) {
		struct dwc3_gadget_ep_cmd_params params;
		struct dwc3_trb	*trb;
		dma_addr_t trb_dma;
		u32 cmd;

		memset(&params, 0, sizeof(params));
		trb = &dep->trb_pool[0];
		trb_dma = dwc3_trb_dma_offset(dep, trb);

		params.param0 = upper_32_bits(trb_dma);
		params.param1 = lower_32_bits(trb_dma);

		cmd = DWC3_DEPCMD_STARTTRANSFER;

		ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
		if (ret < 0)
			return ret;

		dep->flags |= DWC3_EP_BUSY;

		dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
		WARN_ON_ONCE(!dep->resource_index);
	}

700 701 702 703

out:
	trace_dwc3_gadget_ep_enable(dep);

704 705 706
	return 0;
}

707
static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
708
static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
709 710 711
{
	struct dwc3_request		*req;

712
	dwc3_stop_active_transfer(dwc, dep->number, true);
713

714 715 716
	/* - giveback all requests to gadget driver */
	while (!list_empty(&dep->started_list)) {
		req = next_request(&dep->started_list);
717

718
		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
719 720
	}

721 722
	while (!list_empty(&dep->pending_list)) {
		req = next_request(&dep->pending_list);
723

724
		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
725 726 727 728
	}
}

/**
F
Felipe Balbi 已提交
729
 * __dwc3_gadget_ep_disable - disables a hw endpoint
730 731
 * @dep: the endpoint to disable
 *
F
Felipe Balbi 已提交
732 733 734 735
 * This function undoes what __dwc3_gadget_ep_enable did and also removes
 * requests which are currently being processed by the hardware and those which
 * are not yet scheduled.
 *
736
 * Caller should take care of locking.
737 738 739 740 741 742
 */
static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
{
	struct dwc3		*dwc = dep->dwc;
	u32			reg;

743
	trace_dwc3_gadget_ep_disable(dep);
744

745
	dwc3_remove_requests(dwc, dep);
746

747 748
	/* make sure HW endpoint isn't stalled */
	if (dep->flags & DWC3_EP_STALL)
749
		__dwc3_gadget_ep_set_halt(dep, 0, false);
750

751 752 753 754
	reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
	reg &= ~DWC3_DALEPENA_EP(dep->number);
	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);

755
	dep->stream_capable = false;
756
	dep->type = 0;
757
	dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
758

759 760 761 762 763 764
	/* Clear out the ep descriptors for non-ep0 */
	if (dep->number > 1) {
		dep->endpoint.comp_desc = NULL;
		dep->endpoint.desc = NULL;
	}

765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
	return 0;
}

/* -------------------------------------------------------------------------- */

static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
		const struct usb_endpoint_descriptor *desc)
{
	return -EINVAL;
}

static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
{
	return -EINVAL;
}

/* -------------------------------------------------------------------------- */

static int dwc3_gadget_ep_enable(struct usb_ep *ep,
		const struct usb_endpoint_descriptor *desc)
{
	struct dwc3_ep			*dep;
	struct dwc3			*dwc;
	unsigned long			flags;
	int				ret;

	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
		pr_debug("dwc3: invalid parameters\n");
		return -EINVAL;
	}

	if (!desc->wMaxPacketSize) {
		pr_debug("dwc3: missing wMaxPacketSize\n");
		return -EINVAL;
	}

	dep = to_dwc3_ep(ep);
	dwc = dep->dwc;

804 805 806
	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
					"%s is already enabled\n",
					dep->name))
807 808
		return 0;

809
	spin_lock_irqsave(&dwc->lock, flags);
810
	ret = __dwc3_gadget_ep_enable(dep, false, false);
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_ep_disable(struct usb_ep *ep)
{
	struct dwc3_ep			*dep;
	struct dwc3			*dwc;
	unsigned long			flags;
	int				ret;

	if (!ep) {
		pr_debug("dwc3: invalid parameters\n");
		return -EINVAL;
	}

	dep = to_dwc3_ep(ep);
	dwc = dep->dwc;

831 832 833
	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
					"%s is already disabled\n",
					dep->name))
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
		return 0;

	spin_lock_irqsave(&dwc->lock, flags);
	ret = __dwc3_gadget_ep_disable(dep);
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
	gfp_t gfp_flags)
{
	struct dwc3_request		*req;
	struct dwc3_ep			*dep = to_dwc3_ep(ep);

	req = kzalloc(sizeof(*req), gfp_flags);
850
	if (!req)
851 852 853 854 855
		return NULL;

	req->epnum	= dep->number;
	req->dep	= dep;

856 857
	dep->allocated_requests++;

858 859
	trace_dwc3_alloc_request(req);

860 861 862 863 864 865 866
	return &req->request;
}

static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
		struct usb_request *request)
{
	struct dwc3_request		*req = to_dwc3_request(request);
867
	struct dwc3_ep			*dep = to_dwc3_ep(ep);
868

869
	dep->allocated_requests--;
870
	trace_dwc3_free_request(req);
871 872 873
	kfree(req);
}

874 875
static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);

876 877 878
static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
		dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
		unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
879
{
880 881 882
	struct dwc3		*dwc = dep->dwc;
	struct usb_gadget	*gadget = &dwc->gadget;
	enum usb_device_speed	speed = gadget->speed;
883

884
	dwc3_ep_inc_enq(dep);
885

886 887 888
	trb->size = DWC3_TRB_SIZE_LENGTH(length);
	trb->bpl = lower_32_bits(dma);
	trb->bph = upper_32_bits(dma);
889

890
	switch (usb_endpoint_type(dep->endpoint.desc)) {
891
	case USB_ENDPOINT_XFER_CONTROL:
892
		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
893 894 895
		break;

	case USB_ENDPOINT_XFER_ISOC:
896
		if (!node) {
897
			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
898

899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
			/*
			 * USB Specification 2.0 Section 5.9.2 states that: "If
			 * there is only a single transaction in the microframe,
			 * only a DATA0 data packet PID is used.  If there are
			 * two transactions per microframe, DATA1 is used for
			 * the first transaction data packet and DATA0 is used
			 * for the second transaction data packet.  If there are
			 * three transactions per microframe, DATA2 is used for
			 * the first transaction data packet, DATA1 is used for
			 * the second, and DATA0 is used for the third."
			 *
			 * IOW, we should satisfy the following cases:
			 *
			 * 1) length <= maxpacket
			 *	- DATA0
			 *
			 * 2) maxpacket < length <= (2 * maxpacket)
			 *	- DATA1, DATA0
			 *
			 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
			 *	- DATA2, DATA1, DATA0
			 */
921 922
			if (speed == USB_SPEED_HIGH) {
				struct usb_ep *ep = &dep->endpoint;
923 924 925 926 927 928 929 930 931 932
				unsigned int mult = ep->mult - 1;
				unsigned int maxp = usb_endpoint_maxp(ep->desc);

				if (length <= (2 * maxp))
					mult--;

				if (length <= maxp)
					mult--;

				trb->size |= DWC3_TRB_SIZE_PCM1(mult);
933 934
			}
		} else {
935
			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
936
		}
937 938 939

		/* always enable Interrupt on Missed ISOC */
		trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
940 941 942 943
		break;

	case USB_ENDPOINT_XFER_BULK:
	case USB_ENDPOINT_XFER_INT:
944
		trb->ctrl = DWC3_TRBCTL_NORMAL;
945 946 947 948 949 950
		break;
	default:
		/*
		 * This is only possible with faulty memory because we
		 * checked it already :)
		 */
951 952
		dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
				usb_endpoint_type(dep->endpoint.desc));
953 954
	}

955
	/* always enable Continue on Short Packet */
956
	if (usb_endpoint_dir_out(dep->endpoint.desc)) {
957
		trb->ctrl |= DWC3_TRB_CTRL_CSP;
958

959
		if (short_not_ok)
960 961 962
			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
	}

963
	if ((!no_interrupt && !chain) ||
964
			(dwc3_calc_trbs_left(dep) == 0))
965
		trb->ctrl |= DWC3_TRB_CTRL_IOC;
966

967 968 969
	if (chain)
		trb->ctrl |= DWC3_TRB_CTRL_CHN;

970
	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
971
		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
972

973
	trb->ctrl |= DWC3_TRB_CTRL_HWO;
974 975

	trace_dwc3_prepare_trb(dep, trb);
976 977
}

978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
/**
 * dwc3_prepare_one_trb - setup one TRB from one request
 * @dep: endpoint for which this request is prepared
 * @req: dwc3_request pointer
 * @chain: should this TRB be chained to the next?
 * @node: only for isochronous endpoints. First TRB needs different type.
 */
static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
		struct dwc3_request *req, unsigned chain, unsigned node)
{
	struct dwc3_trb		*trb;
	unsigned		length = req->request.length;
	unsigned		stream_id = req->request.stream_id;
	unsigned		short_not_ok = req->request.short_not_ok;
	unsigned		no_interrupt = req->request.no_interrupt;
	dma_addr_t		dma = req->request.dma;

	trb = &dep->trb_pool[dep->trb_enqueue];

	if (!req->trb) {
		dwc3_gadget_move_started_request(req);
		req->trb = trb;
		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
		dep->queued_requests++;
	}

	__dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
			stream_id, short_not_ok, no_interrupt);
}

1008
/**
F
Felipe Balbi 已提交
1009
 * dwc3_ep_prev_trb - returns the previous TRB in the ring
1010 1011 1012 1013 1014 1015 1016 1017 1018
 * @dep: The endpoint with the TRB ring
 * @index: The index of the current TRB in the ring
 *
 * Returns the TRB prior to the one pointed to by the index. If the
 * index is 0, we will wrap backwards, skip the link TRB, and return
 * the one just before that.
 */
static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
{
1019
	u8 tmp = index;
1020

1021 1022
	if (!tmp)
		tmp = DWC3_TRB_NUM - 1;
1023

1024
	return &dep->trb_pool[tmp - 1];
1025 1026
}

1027 1028 1029
static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
{
	struct dwc3_trb		*tmp;
1030
	u8			trbs_left;
1031 1032 1033 1034 1035 1036 1037 1038 1039

	/*
	 * If enqueue & dequeue are equal than it is either full or empty.
	 *
	 * One way to know for sure is if the TRB right before us has HWO bit
	 * set or not. If it has, then we're definitely full and can't fit any
	 * more transfers in our ring.
	 */
	if (dep->trb_enqueue == dep->trb_dequeue) {
1040
		tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1041
		if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
1042
			return 0;
1043 1044 1045 1046

		return DWC3_TRB_NUM - 1;
	}

1047
	trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1048
	trbs_left &= (DWC3_TRB_NUM - 1);
1049

1050 1051 1052
	if (dep->trb_dequeue < dep->trb_enqueue)
		trbs_left--;

1053
	return trbs_left;
1054 1055
}

1056
static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1057
		struct dwc3_request *req)
1058
{
1059
	struct scatterlist *sg = req->sg;
1060 1061 1062
	struct scatterlist *s;
	int		i;

1063
	for_each_sg(sg, s, req->num_pending_sgs, i) {
1064 1065 1066
		unsigned int length = req->request.length;
		unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
		unsigned int rem = length % maxp;
1067 1068
		unsigned chain = true;

1069
		if (sg_is_last(s))
1070 1071
			chain = false;

1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
		if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
			struct dwc3	*dwc = dep->dwc;
			struct dwc3_trb	*trb;

			req->unaligned = true;

			/* prepare normal TRB */
			dwc3_prepare_one_trb(dep, req, true, i);

			/* Now prepare one extra TRB to align transfer size */
			trb = &dep->trb_pool[dep->trb_enqueue];
			__dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
					maxp - rem, false, 0,
					req->request.stream_id,
					req->request.short_not_ok,
					req->request.no_interrupt);
		} else {
			dwc3_prepare_one_trb(dep, req, chain, i);
		}
1091

1092
		if (!dwc3_calc_trbs_left(dep))
1093 1094 1095 1096 1097
			break;
	}
}

static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1098
		struct dwc3_request *req)
1099
{
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
	unsigned int length = req->request.length;
	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
	unsigned int rem = length % maxp;

	if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
		struct dwc3	*dwc = dep->dwc;
		struct dwc3_trb	*trb;

		req->unaligned = true;

		/* prepare normal TRB */
		dwc3_prepare_one_trb(dep, req, true, 0);

		/* Now prepare one extra TRB to align transfer size */
		trb = &dep->trb_pool[dep->trb_enqueue];
		__dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
				false, 0, req->request.stream_id,
				req->request.short_not_ok,
				req->request.no_interrupt);
F
Felipe Balbi 已提交
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
	} else if (req->request.zero && req->request.length &&
		   (IS_ALIGNED(req->request.length,dep->endpoint.maxpacket))) {
		struct dwc3	*dwc = dep->dwc;
		struct dwc3_trb	*trb;

		req->zero = true;

		/* prepare normal TRB */
		dwc3_prepare_one_trb(dep, req, true, 0);

		/* Now prepare one extra TRB to handle ZLP */
		trb = &dep->trb_pool[dep->trb_enqueue];
		__dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
				false, 0, req->request.stream_id,
				req->request.short_not_ok,
				req->request.no_interrupt);
1135 1136 1137
	} else {
		dwc3_prepare_one_trb(dep, req, false, 0);
	}
1138 1139
}

1140 1141 1142 1143
/*
 * dwc3_prepare_trbs - setup TRBs from requests
 * @dep: endpoint for which requests are being prepared
 *
1144 1145 1146
 * The function goes through the requests list and sets up TRBs for the
 * transfers. The function returns once there are no more TRBs available or
 * it runs out of requests.
1147
 */
1148
static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1149
{
1150
	struct dwc3_request	*req, *n;
1151 1152 1153

	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);

1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
	/*
	 * We can get in a situation where there's a request in the started list
	 * but there weren't enough TRBs to fully kick it in the first time
	 * around, so it has been waiting for more TRBs to be freed up.
	 *
	 * In that case, we should check if we have a request with pending_sgs
	 * in the started list and prepare TRBs for that request first,
	 * otherwise we will prepare TRBs completely out of order and that will
	 * break things.
	 */
	list_for_each_entry(req, &dep->started_list, list) {
		if (req->num_pending_sgs > 0)
			dwc3_prepare_one_trb_sg(dep, req);

		if (!dwc3_calc_trbs_left(dep))
			return;
	}

1172
	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
		struct dwc3	*dwc = dep->dwc;
		int		ret;

		ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
						    dep->direction);
		if (ret)
			return;

		req->sg			= req->request.sg;
		req->num_pending_sgs	= req->request.num_mapped_sgs;

1184
		if (req->num_pending_sgs > 0)
1185
			dwc3_prepare_one_trb_sg(dep, req);
1186
		else
1187
			dwc3_prepare_one_trb_linear(dep, req);
1188

1189
		if (!dwc3_calc_trbs_left(dep))
1190
			return;
1191 1192 1193
	}
}

1194
static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1195 1196 1197
{
	struct dwc3_gadget_ep_cmd_params params;
	struct dwc3_request		*req;
1198
	int				starting;
1199 1200 1201
	int				ret;
	u32				cmd;

1202 1203 1204
	if (!dwc3_calc_trbs_left(dep))
		return 0;

1205
	starting = !(dep->flags & DWC3_EP_BUSY);
1206

1207 1208
	dwc3_prepare_trbs(dep);
	req = next_request(&dep->started_list);
1209 1210 1211 1212 1213 1214 1215
	if (!req) {
		dep->flags |= DWC3_EP_PENDING_REQUEST;
		return 0;
	}

	memset(&params, 0, sizeof(params));

1216
	if (starting) {
1217 1218
		params.param0 = upper_32_bits(req->trb_dma);
		params.param1 = lower_32_bits(req->trb_dma);
1219 1220 1221 1222
		cmd = DWC3_DEPCMD_STARTTRANSFER;

		if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
			cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1223
	} else {
1224 1225
		cmd = DWC3_DEPCMD_UPDATETRANSFER |
			DWC3_DEPCMD_PARAM(dep->resource_index);
1226
	}
1227

1228
	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1229 1230 1231 1232
	if (ret < 0) {
		/*
		 * FIXME we need to iterate over the list of requests
		 * here and stop, unmap, free and del each of the linked
1233
		 * requests instead of what we do now.
1234
		 */
1235 1236
		if (req->trb)
			memset(req->trb, 0, sizeof(struct dwc3_trb));
1237
		dep->queued_requests--;
1238
		dwc3_gadget_giveback(dep, req, ret);
1239 1240 1241 1242
		return ret;
	}

	dep->flags |= DWC3_EP_BUSY;
1243

1244
	if (starting) {
1245
		dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
1246
		WARN_ON_ONCE(!dep->resource_index);
1247
	}
1248

1249 1250 1251
	return 0;
}

1252 1253 1254 1255 1256 1257 1258 1259
static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
{
	u32			reg;

	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
	return DWC3_DSTS_SOFFN(reg);
}

1260 1261 1262
static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
		struct dwc3_ep *dep, u32 cur_uf)
{
1263
	if (list_empty(&dep->pending_list)) {
1264
		dev_info(dwc->dev, "%s: ran out of requests\n",
1265
				dep->name);
1266
		dep->flags |= DWC3_EP_PENDING_REQUEST;
1267 1268 1269
		return;
	}

1270 1271 1272 1273
	/*
	 * Schedule the first trb for one interval in the future or at
	 * least 4 microframes.
	 */
1274
	dep->frame_number = cur_uf + max_t(u32, 4, dep->interval);
1275
	__dwc3_gadget_kick_transfer(dep);
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
}

static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
{
	u32 cur_uf, mask;

	mask = ~(dep->interval - 1);
	cur_uf = event->parameters & mask;

	__dwc3_gadget_start_isoc(dwc, dep, cur_uf);
}

1289 1290
static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
{
1291 1292
	struct dwc3		*dwc = dep->dwc;

1293
	if (!dep->endpoint.desc) {
1294 1295
		dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
				dep->name);
1296 1297 1298
		return -ESHUTDOWN;
	}

F
Felipe Balbi 已提交
1299 1300
	if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
				&req->request, req->dep->name))
1301 1302
		return -EINVAL;

F
Felipe Balbi 已提交
1303 1304
	pm_runtime_get(dwc->dev);

1305 1306 1307 1308 1309
	req->request.actual	= 0;
	req->request.status	= -EINPROGRESS;
	req->direction		= dep->direction;
	req->epnum		= dep->number;

1310 1311
	trace_dwc3_ep_queue(req);

1312
	list_add_tail(&req->list, &dep->pending_list);
1313

1314 1315 1316 1317 1318 1319 1320 1321 1322
	/*
	 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
	 * wait for a XferNotReady event so we will know what's the current
	 * (micro-)frame number.
	 *
	 * Without this trick, we are very, very likely gonna get Bus Expiry
	 * errors which will force us issue EndTransfer command.
	 */
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1323 1324 1325 1326 1327 1328 1329 1330 1331
		if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
			if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
				dwc3_stop_active_transfer(dwc, dep->number, true);
				dep->flags = DWC3_EP_ENABLED;
			} else {
				u32 cur_uf;

				cur_uf = __dwc3_gadget_get_frame(dwc);
				__dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1332
				dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1333
			}
1334
			return 0;
1335
		}
1336 1337

		if ((dep->flags & DWC3_EP_BUSY) &&
1338 1339
		    !(dep->flags & DWC3_EP_MISSED_ISOC))
			goto out;
1340

1341
		return 0;
1342
	}
1343

1344
out:
1345
	return __dwc3_gadget_kick_transfer(dep);
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
}

static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
	gfp_t gfp_flags)
{
	struct dwc3_request		*req = to_dwc3_request(request);
	struct dwc3_ep			*dep = to_dwc3_ep(ep);
	struct dwc3			*dwc = dep->dwc;

	unsigned long			flags;

	int				ret;

1359
	spin_lock_irqsave(&dwc->lock, flags);
1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
	ret = __dwc3_gadget_ep_queue(dep, req);
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
		struct usb_request *request)
{
	struct dwc3_request		*req = to_dwc3_request(request);
	struct dwc3_request		*r = NULL;

	struct dwc3_ep			*dep = to_dwc3_ep(ep);
	struct dwc3			*dwc = dep->dwc;

	unsigned long			flags;
	int				ret = 0;

1378 1379
	trace_dwc3_ep_dequeue(req);

1380 1381
	spin_lock_irqsave(&dwc->lock, flags);

1382
	list_for_each_entry(r, &dep->pending_list, list) {
1383 1384 1385 1386 1387
		if (r == req)
			break;
	}

	if (r != req) {
1388
		list_for_each_entry(r, &dep->started_list, list) {
1389 1390 1391 1392 1393
			if (r == req)
				break;
		}
		if (r == req) {
			/* wait until it is processed */
1394
			dwc3_stop_active_transfer(dwc, dep->number, true);
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439

			/*
			 * If request was already started, this means we had to
			 * stop the transfer. With that we also need to ignore
			 * all TRBs used by the request, however TRBs can only
			 * be modified after completion of END_TRANSFER
			 * command. So what we do here is that we wait for
			 * END_TRANSFER completion and only after that, we jump
			 * over TRBs by clearing HWO and incrementing dequeue
			 * pointer.
			 *
			 * Note that we have 2 possible types of transfers here:
			 *
			 * i) Linear buffer request
			 * ii) SG-list based request
			 *
			 * SG-list based requests will have r->num_pending_sgs
			 * set to a valid number (> 0). Linear requests,
			 * normally use a single TRB.
			 *
			 * For each of these two cases, if r->unaligned flag is
			 * set, one extra TRB has been used to align transfer
			 * size to wMaxPacketSize.
			 *
			 * All of these cases need to be taken into
			 * consideration so we don't mess up our TRB ring
			 * pointers.
			 */
			wait_event_lock_irq(dep->wait_end_transfer,
					!(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
					dwc->lock);

			if (!r->trb)
				goto out1;

			if (r->num_pending_sgs) {
				struct dwc3_trb *trb;
				int i = 0;

				for (i = 0; i < r->num_pending_sgs; i++) {
					trb = r->trb + i;
					trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
					dwc3_ep_inc_deq(dep);
				}

F
Felipe Balbi 已提交
1440
				if (r->unaligned || r->zero) {
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
					trb = r->trb + r->num_pending_sgs + 1;
					trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
					dwc3_ep_inc_deq(dep);
				}
			} else {
				struct dwc3_trb *trb = r->trb;

				trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
				dwc3_ep_inc_deq(dep);

F
Felipe Balbi 已提交
1451
				if (r->unaligned || r->zero) {
1452 1453 1454 1455 1456
					trb = r->trb + 1;
					trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
					dwc3_ep_inc_deq(dep);
				}
			}
1457
			goto out1;
1458
		}
F
Felipe Balbi 已提交
1459
		dev_err(dwc->dev, "request %pK was not queued to %s\n",
1460 1461 1462 1463 1464
				request, ep->name);
		ret = -EINVAL;
		goto out0;
	}

1465
out1:
1466
	/* giveback the request */
1467
	dep->queued_requests--;
1468 1469 1470 1471 1472 1473 1474 1475
	dwc3_gadget_giveback(dep, req, -ECONNRESET);

out0:
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

1476
int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1477 1478 1479 1480 1481
{
	struct dwc3_gadget_ep_cmd_params	params;
	struct dwc3				*dwc = dep->dwc;
	int					ret;

1482 1483 1484 1485 1486
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
		return -EINVAL;
	}

1487 1488 1489
	memset(&params, 0x00, sizeof(params));

	if (value) {
1490 1491 1492 1493 1494
		struct dwc3_trb *trb;

		unsigned transfer_in_flight;
		unsigned started;

1495 1496 1497
		if (dep->flags & DWC3_EP_STALL)
			return 0;

1498 1499 1500 1501 1502 1503 1504 1505 1506 1507
		if (dep->number > 1)
			trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
		else
			trb = &dwc->ep0_trb[dep->trb_enqueue];

		transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
		started = !list_empty(&dep->started_list);

		if (!protocol && ((dep->direction && transfer_in_flight) ||
				(!dep->direction && started))) {
1508 1509 1510
			return -EAGAIN;
		}

1511 1512
		ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
				&params);
1513
		if (ret)
1514
			dev_err(dwc->dev, "failed to set STALL on %s\n",
1515 1516 1517 1518
					dep->name);
		else
			dep->flags |= DWC3_EP_STALL;
	} else {
1519 1520
		if (!(dep->flags & DWC3_EP_STALL))
			return 0;
1521

1522
		ret = dwc3_send_clear_stall_ep_cmd(dep);
1523
		if (ret)
1524
			dev_err(dwc->dev, "failed to clear STALL on %s\n",
1525 1526
					dep->name);
		else
1527
			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1528
	}
1529

1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542
	return ret;
}

static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
{
	struct dwc3_ep			*dep = to_dwc3_ep(ep);
	struct dwc3			*dwc = dep->dwc;

	unsigned long			flags;

	int				ret;

	spin_lock_irqsave(&dwc->lock, flags);
1543
	ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1544 1545 1546 1547 1548 1549 1550 1551
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
{
	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1552 1553
	struct dwc3			*dwc = dep->dwc;
	unsigned long			flags;
1554
	int				ret;
1555

1556
	spin_lock_irqsave(&dwc->lock, flags);
1557 1558
	dep->flags |= DWC3_EP_WEDGE;

1559
	if (dep->number == 0 || dep->number == 1)
1560
		ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1561
	else
1562
		ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1563 1564 1565
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
}

/* -------------------------------------------------------------------------- */

static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
	.bLength	= USB_DT_ENDPOINT_SIZE,
	.bDescriptorType = USB_DT_ENDPOINT,
	.bmAttributes	= USB_ENDPOINT_XFER_CONTROL,
};

static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
	.enable		= dwc3_gadget_ep0_enable,
	.disable	= dwc3_gadget_ep0_disable,
	.alloc_request	= dwc3_gadget_ep_alloc_request,
	.free_request	= dwc3_gadget_ep_free_request,
	.queue		= dwc3_gadget_ep0_queue,
	.dequeue	= dwc3_gadget_ep_dequeue,
1583
	.set_halt	= dwc3_gadget_ep0_set_halt,
1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
	.set_wedge	= dwc3_gadget_ep_set_wedge,
};

static const struct usb_ep_ops dwc3_gadget_ep_ops = {
	.enable		= dwc3_gadget_ep_enable,
	.disable	= dwc3_gadget_ep_disable,
	.alloc_request	= dwc3_gadget_ep_alloc_request,
	.free_request	= dwc3_gadget_ep_free_request,
	.queue		= dwc3_gadget_ep_queue,
	.dequeue	= dwc3_gadget_ep_dequeue,
	.set_halt	= dwc3_gadget_ep_set_halt,
	.set_wedge	= dwc3_gadget_ep_set_wedge,
};

/* -------------------------------------------------------------------------- */

static int dwc3_gadget_get_frame(struct usb_gadget *g)
{
	struct dwc3		*dwc = gadget_to_dwc(g);

1604
	return __dwc3_gadget_get_frame(dwc);
1605 1606
}

1607
static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1608
{
1609
	int			retries;
1610

1611
	int			ret;
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625
	u32			reg;

	u8			link_state;
	u8			speed;

	/*
	 * According to the Databook Remote wakeup request should
	 * be issued only when the device is in early suspend state.
	 *
	 * We can check that via USB Link State bits in DSTS register.
	 */
	reg = dwc3_readl(dwc->regs, DWC3_DSTS);

	speed = reg & DWC3_DSTS_CONNECTSPD;
1626
	if ((speed == DWC3_DSTS_SUPERSPEED) ||
1627
	    (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1628
		return 0;
1629 1630 1631 1632 1633 1634 1635 1636

	link_state = DWC3_DSTS_USBLNKST(reg);

	switch (link_state) {
	case DWC3_LINK_STATE_RX_DET:	/* in HS, means Early Suspend */
	case DWC3_LINK_STATE_U3:	/* in HS, means SUSPEND */
		break;
	default:
1637
		return -EINVAL;
1638 1639
	}

1640 1641 1642
	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
	if (ret < 0) {
		dev_err(dwc->dev, "failed to put link in Recovery\n");
1643
		return ret;
1644
	}
1645

1646 1647 1648
	/* Recent versions do this automatically */
	if (dwc->revision < DWC3_REVISION_194A) {
		/* write zeroes to Link Change Request */
1649
		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1650 1651 1652
		reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	}
1653

1654
	/* poll until Link State changes to ON */
1655
	retries = 20000;
1656

1657
	while (retries--) {
1658 1659 1660 1661 1662 1663 1664 1665 1666
		reg = dwc3_readl(dwc->regs, DWC3_DSTS);

		/* in HS, means ON */
		if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
			break;
	}

	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
		dev_err(dwc->dev, "failed to send remote wakeup\n");
1667
		return -EINVAL;
1668 1669
	}

1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
	return 0;
}

static int dwc3_gadget_wakeup(struct usb_gadget *g)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	unsigned long		flags;
	int			ret;

	spin_lock_irqsave(&dwc->lock, flags);
	ret = __dwc3_gadget_wakeup(dwc);
1681 1682 1683 1684 1685 1686 1687 1688 1689
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
		int is_selfpowered)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
1690
	unsigned long		flags;
1691

1692
	spin_lock_irqsave(&dwc->lock, flags);
1693
	g->is_selfpowered = !!is_selfpowered;
1694
	spin_unlock_irqrestore(&dwc->lock, flags);
1695 1696 1697 1698

	return 0;
}

1699
static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1700 1701
{
	u32			reg;
1702
	u32			timeout = 500;
1703

F
Felipe Balbi 已提交
1704 1705 1706
	if (pm_runtime_suspended(dwc->dev))
		return 0;

1707
	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1708
	if (is_on) {
1709 1710 1711 1712 1713 1714 1715 1716
		if (dwc->revision <= DWC3_REVISION_187A) {
			reg &= ~DWC3_DCTL_TRGTULST_MASK;
			reg |= DWC3_DCTL_TRGTULST_RX_DET;
		}

		if (dwc->revision >= DWC3_REVISION_194A)
			reg &= ~DWC3_DCTL_KEEP_CONNECT;
		reg |= DWC3_DCTL_RUN_STOP;
1717 1718 1719 1720

		if (dwc->has_hibernation)
			reg |= DWC3_DCTL_KEEP_CONNECT;

1721
		dwc->pullups_connected = true;
1722
	} else {
1723
		reg &= ~DWC3_DCTL_RUN_STOP;
1724 1725 1726 1727

		if (dwc->has_hibernation && !suspend)
			reg &= ~DWC3_DCTL_KEEP_CONNECT;

1728
		dwc->pullups_connected = false;
1729
	}
1730 1731 1732 1733 1734

	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	do {
		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1735 1736
		reg &= DWC3_DSTS_DEVCTRLHLT;
	} while (--timeout && !(!is_on ^ !reg));
1737 1738 1739

	if (!timeout)
		return -ETIMEDOUT;
1740

1741
	return 0;
1742 1743 1744 1745 1746 1747
}

static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	unsigned long		flags;
1748
	int			ret;
1749 1750 1751

	is_on = !!is_on;

1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766
	/*
	 * Per databook, when we want to stop the gadget, if a control transfer
	 * is still in process, complete it and get the core into setup phase.
	 */
	if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
		reinit_completion(&dwc->ep0_in_setup);

		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
				msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
		if (ret == 0) {
			dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
			return -ETIMEDOUT;
		}
	}

1767
	spin_lock_irqsave(&dwc->lock, flags);
1768
	ret = dwc3_gadget_run_stop(dwc, is_on, false);
1769 1770
	spin_unlock_irqrestore(&dwc->lock, flags);

1771
	return ret;
1772 1773
}

1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
{
	u32			reg;

	/* Enable all but Start and End of Frame IRQs */
	reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
			DWC3_DEVTEN_EVNTOVERFLOWEN |
			DWC3_DEVTEN_CMDCMPLTEN |
			DWC3_DEVTEN_ERRTICERREN |
			DWC3_DEVTEN_WKUPEVTEN |
			DWC3_DEVTEN_CONNECTDONEEN |
			DWC3_DEVTEN_USBRSTEN |
			DWC3_DEVTEN_DISCONNEVTEN);

1788 1789 1790
	if (dwc->revision < DWC3_REVISION_250A)
		reg |= DWC3_DEVTEN_ULSTCNGEN;

1791 1792 1793 1794 1795 1796 1797 1798 1799 1800
	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
}

static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
{
	/* mask all interrupts */
	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
}

static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1801
static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1802

1803
/**
F
Felipe Balbi 已提交
1804 1805
 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
 * @dwc: pointer to our context structure
1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843
 *
 * The following looks like complex but it's actually very simple. In order to
 * calculate the number of packets we can burst at once on OUT transfers, we're
 * gonna use RxFIFO size.
 *
 * To calculate RxFIFO size we need two numbers:
 * MDWIDTH = size, in bits, of the internal memory bus
 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
 *
 * Given these two numbers, the formula is simple:
 *
 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
 *
 * 24 bytes is for 3x SETUP packets
 * 16 bytes is a clock domain crossing tolerance
 *
 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
 */
static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
{
	u32 ram2_depth;
	u32 mdwidth;
	u32 nump;
	u32 reg;

	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
	mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);

	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
	nump = min_t(u32, nump, 16);

	/* update NumP */
	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	reg &= ~DWC3_DCFG_NUMP_MASK;
	reg |= nump << DWC3_DCFG_NUMP_SHIFT;
	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
}

1844
static int __dwc3_gadget_start(struct dwc3 *dwc)
1845 1846 1847 1848 1849
{
	struct dwc3_ep		*dep;
	int			ret = 0;
	u32			reg;

1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
	/*
	 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
	 * the core supports IMOD, disable it.
	 */
	if (dwc->imod_interval) {
		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
	} else if (dwc3_has_imod(dwc)) {
		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
	}

1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
	/*
	 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
	 * field instead of letting dwc3 itself calculate that automatically.
	 *
	 * This way, we maximize the chances that we'll be able to get several
	 * bursts of data without going through any sort of endpoint throttling.
	 */
	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
	reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);

1872 1873
	dwc3_gadget_setup_nump(dwc);

1874 1875 1876 1877
	/* Start with SuperSpeed Default */
	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);

	dep = dwc->eps[0];
1878
	ret = __dwc3_gadget_ep_enable(dep, false, false);
1879 1880
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1881
		goto err0;
1882 1883 1884
	}

	dep = dwc->eps[1];
1885
	ret = __dwc3_gadget_ep_enable(dep, false, false);
1886 1887
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1888
		goto err1;
1889 1890 1891
	}

	/* begin to receive SETUP packets */
1892
	dwc->ep0state = EP0_SETUP_PHASE;
1893 1894
	dwc3_ep0_out_start(dwc);

1895 1896
	dwc3_gadget_enable_irq(dwc);

1897 1898
	return 0;

1899
err1:
1900
	__dwc3_gadget_ep_disable(dwc->eps[0]);
1901 1902

err0:
1903 1904 1905
	return ret;
}

1906 1907
static int dwc3_gadget_start(struct usb_gadget *g,
		struct usb_gadget_driver *driver)
1908 1909 1910
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	unsigned long		flags;
1911
	int			ret = 0;
1912
	int			irq;
1913

1914
	irq = dwc->irq_gadget;
1915 1916 1917 1918 1919 1920 1921 1922
	ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
			IRQF_SHARED, "dwc3", dwc->ev_buf);
	if (ret) {
		dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
				irq, ret);
		goto err0;
	}

1923
	spin_lock_irqsave(&dwc->lock, flags);
1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
	if (dwc->gadget_driver) {
		dev_err(dwc->dev, "%s is already bound to %s\n",
				dwc->gadget.name,
				dwc->gadget_driver->driver.name);
		ret = -EBUSY;
		goto err1;
	}

	dwc->gadget_driver	= driver;

F
Felipe Balbi 已提交
1934 1935 1936
	if (pm_runtime_active(dwc->dev))
		__dwc3_gadget_start(dwc);

1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947
	spin_unlock_irqrestore(&dwc->lock, flags);

	return 0;

err1:
	spin_unlock_irqrestore(&dwc->lock, flags);
	free_irq(irq, dwc);

err0:
	return ret;
}
1948

1949 1950
static void __dwc3_gadget_stop(struct dwc3 *dwc)
{
1951
	dwc3_gadget_disable_irq(dwc);
1952 1953
	__dwc3_gadget_ep_disable(dwc->eps[0]);
	__dwc3_gadget_ep_disable(dwc->eps[1]);
1954
}
1955

1956 1957 1958 1959
static int dwc3_gadget_stop(struct usb_gadget *g)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	unsigned long		flags;
1960
	int			epnum;
1961

1962
	spin_lock_irqsave(&dwc->lock, flags);
1963 1964 1965 1966

	if (pm_runtime_suspended(dwc->dev))
		goto out;

1967
	__dwc3_gadget_stop(dwc);
1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983

	for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
		struct dwc3_ep  *dep = dwc->eps[epnum];

		if (!dep)
			continue;

		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
			continue;

		wait_event_lock_irq(dep->wait_end_transfer,
				    !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
				    dwc->lock);
	}

out:
1984
	dwc->gadget_driver	= NULL;
1985 1986
	spin_unlock_irqrestore(&dwc->lock, flags);

1987
	free_irq(dwc->irq_gadget, dwc->ev_buf);
1988

1989 1990
	return 0;
}
1991

1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
static void dwc3_gadget_set_speed(struct usb_gadget *g,
				  enum usb_device_speed speed)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	unsigned long		flags;
	u32			reg;

	spin_lock_irqsave(&dwc->lock, flags);
	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	reg &= ~(DWC3_DCFG_SPEED_MASK);

	/*
	 * WORKAROUND: DWC3 revision < 2.20a have an issue
	 * which would cause metastability state on Run/Stop
	 * bit if we try to force the IP to USB2-only mode.
	 *
	 * Because of that, we cannot configure the IP to any
	 * speed other than the SuperSpeed
	 *
	 * Refers to:
	 *
	 * STAR#9000525659: Clock Domain Crossing on DCTL in
	 * USB 2.0 Mode
	 */
	if (dwc->revision < DWC3_REVISION_220A) {
		reg |= DWC3_DCFG_SUPERSPEED;
	} else {
		switch (speed) {
		case USB_SPEED_LOW:
			reg |= DWC3_DCFG_LOWSPEED;
			break;
		case USB_SPEED_FULL:
			reg |= DWC3_DCFG_FULLSPEED;
			break;
		case USB_SPEED_HIGH:
			reg |= DWC3_DCFG_HIGHSPEED;
			break;
		case USB_SPEED_SUPER:
			reg |= DWC3_DCFG_SUPERSPEED;
			break;
		case USB_SPEED_SUPER_PLUS:
			reg |= DWC3_DCFG_SUPERSPEED_PLUS;
			break;
		default:
			dev_err(dwc->dev, "invalid speed (%d)\n", speed);

			if (dwc->revision & DWC3_REVISION_IS_DWC31)
				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
			else
				reg |= DWC3_DCFG_SUPERSPEED;
		}
	}
	dwc3_writel(dwc->regs, DWC3_DCFG, reg);

	spin_unlock_irqrestore(&dwc->lock, flags);
}

2049 2050 2051 2052 2053 2054 2055
static const struct usb_gadget_ops dwc3_gadget_ops = {
	.get_frame		= dwc3_gadget_get_frame,
	.wakeup			= dwc3_gadget_wakeup,
	.set_selfpowered	= dwc3_gadget_set_selfpowered,
	.pullup			= dwc3_gadget_pullup,
	.udc_start		= dwc3_gadget_start,
	.udc_stop		= dwc3_gadget_stop,
2056
	.udc_set_speed		= dwc3_gadget_set_speed,
2057 2058 2059 2060
};

/* -------------------------------------------------------------------------- */

2061
static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2062 2063
{
	struct dwc3_ep			*dep;
2064
	u8				epnum;
2065

2066 2067
	INIT_LIST_HEAD(&dwc->gadget.ep_list);

2068
	for (epnum = 0; epnum < total; epnum++) {
2069
		bool			direction = epnum & 1;
2070
		u8			num = epnum >> 1;
2071 2072

		dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2073
		if (!dep)
2074 2075 2076 2077
			return -ENOMEM;

		dep->dwc = dwc;
		dep->number = epnum;
2078
		dep->direction = direction;
2079
		dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2080 2081
		dwc->eps[epnum] = dep;

2082
		snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2083
				direction ? "in" : "out");
2084

2085
		dep->endpoint.name = dep->name;
2086 2087 2088 2089 2090 2091

		if (!(dep->number > 1)) {
			dep->endpoint.desc = &dwc3_gadget_ep0_desc;
			dep->endpoint.comp_desc = NULL;
		}

2092
		spin_lock_init(&dep->lock);
2093

2094
		if (num == 0) {
2095
			usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2096
			dep->endpoint.maxburst = 1;
2097
			dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2098
			if (!direction)
2099
				dwc->gadget.ep0 = &dep->endpoint;
2100 2101
		} else if (direction) {
			int mdwidth;
2102
			int kbytes;
2103 2104 2105 2106 2107 2108 2109
			int size;
			int ret;

			mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
			/* MDWIDTH is represented in bits, we need it in bytes */
			mdwidth /= 8;

2110
			size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num));
2111 2112 2113 2114 2115
			size = DWC3_GTXFIFOSIZ_TXFDEF(size);

			/* FIFO Depth is in MDWDITH bytes. Multiply */
			size *= mdwidth;

2116 2117 2118
			kbytes = size / 1024;
			if (kbytes == 0)
				kbytes = 1;
2119 2120

			/*
2121
			 * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
2122 2123 2124
			 * internal overhead. We don't really know how these are used,
			 * but documentation say it exists.
			 */
2125 2126
			size -= mdwidth * (kbytes + 1);
			size /= kbytes;
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137

			usb_ep_set_maxpacket_limit(&dep->endpoint, size);

			dep->endpoint.max_streams = 15;
			dep->endpoint.ops = &dwc3_gadget_ep_ops;
			list_add_tail(&dep->endpoint.ep_list,
					&dwc->gadget.ep_list);

			ret = dwc3_alloc_trb_pool(dep);
			if (ret)
				return ret;
2138 2139 2140
		} else {
			int		ret;

2141
			usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2142
			dep->endpoint.max_streams = 15;
2143 2144 2145 2146 2147
			dep->endpoint.ops = &dwc3_gadget_ep_ops;
			list_add_tail(&dep->endpoint.ep_list,
					&dwc->gadget.ep_list);

			ret = dwc3_alloc_trb_pool(dep);
2148
			if (ret)
2149 2150
				return ret;
		}
2151

2152
		if (num == 0) {
2153 2154 2155 2156 2157 2158 2159
			dep->endpoint.caps.type_control = true;
		} else {
			dep->endpoint.caps.type_iso = true;
			dep->endpoint.caps.type_bulk = true;
			dep->endpoint.caps.type_int = true;
		}

2160
		dep->endpoint.caps.dir_in = direction;
2161 2162
		dep->endpoint.caps.dir_out = !direction;

2163 2164
		INIT_LIST_HEAD(&dep->pending_list);
		INIT_LIST_HEAD(&dep->started_list);
2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
	}

	return 0;
}

static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
{
	struct dwc3_ep			*dep;
	u8				epnum;

	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
		dep = dwc->eps[epnum];
2177 2178
		if (!dep)
			continue;
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189
		/*
		 * Physical endpoints 0 and 1 are special; they form the
		 * bi-directional USB endpoint 0.
		 *
		 * For those two physical endpoints, we don't allocate a TRB
		 * pool nor do we add them the endpoints list. Due to that, we
		 * shouldn't do these two operations otherwise we would end up
		 * with all sorts of bugs when removing dwc3.ko.
		 */
		if (epnum != 0 && epnum != 1) {
			dwc3_free_trb_pool(dep);
2190
			list_del(&dep->endpoint.ep_list);
2191
		}
2192 2193 2194 2195 2196 2197

		kfree(dep);
	}
}

/* -------------------------------------------------------------------------- */
2198

2199 2200
static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
		struct dwc3_request *req, struct dwc3_trb *trb,
2201 2202
		const struct dwc3_event_depevt *event, int status,
		int chain)
2203 2204 2205
{
	unsigned int		count;
	unsigned int		s_pkt = 0;
2206
	unsigned int		trb_status;
2207

2208
	dwc3_ep_inc_deq(dep);
2209 2210 2211 2212

	if (req->trb == trb)
		dep->queued_requests--;

2213 2214
	trace_dwc3_complete_trb(dep, trb);

2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
	/*
	 * If we're in the middle of series of chained TRBs and we
	 * receive a short transfer along the way, DWC3 will skip
	 * through all TRBs including the last TRB in the chain (the
	 * where CHN bit is zero. DWC3 will also avoid clearing HWO
	 * bit and SW has to do it manually.
	 *
	 * We're going to do that here to avoid problems of HW trying
	 * to use bogus TRBs for transfers.
	 */
	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;

2228 2229 2230 2231 2232
	/*
	 * If we're dealing with unaligned size OUT transfer, we will be left
	 * with one TRB pending in the ring. We need to manually clear HWO bit
	 * from that TRB.
	 */
F
Felipe Balbi 已提交
2233
	if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
2234 2235 2236 2237
		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
		return 1;
	}

2238
	count = trb->size & DWC3_TRB_SIZE_MASK;
2239
	req->remaining += count;
2240

2241 2242 2243
	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
		return 1;

2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
	if (dep->direction) {
		if (count) {
			trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
			if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
				/*
				 * If missed isoc occurred and there is
				 * no request queued then issue END
				 * TRANSFER, so that core generates
				 * next xfernotready and we will issue
				 * a fresh START TRANSFER.
				 * If there are still queued request
				 * then wait, do not issue either END
				 * or UPDATE TRANSFER, just attach next
2257
				 * request in pending_list during
2258 2259 2260
				 * giveback.If any future queued request
				 * is successfully transferred then we
				 * will issue UPDATE TRANSFER for all
2261
				 * request in the pending_list.
2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276
				 */
				dep->flags |= DWC3_EP_MISSED_ISOC;
			} else {
				dev_err(dwc->dev, "incomplete IN transfer %s\n",
						dep->name);
				status = -ECONNRESET;
			}
		} else {
			dep->flags &= ~DWC3_EP_MISSED_ISOC;
		}
	} else {
		if (count && (event->status & DEPEVT_STATUS_SHORT))
			s_pkt = 1;
	}

2277
	if (s_pkt && !chain)
2278
		return 1;
2279

2280 2281 2282
	if ((event->status & DEPEVT_STATUS_IOC) &&
			(trb->ctrl & DWC3_TRB_CTRL_IOC))
		return 1;
2283

2284 2285 2286 2287 2288 2289
	return 0;
}

static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
		const struct dwc3_event_depevt *event, int status)
{
2290
	struct dwc3_request	*req, *n;
2291
	struct dwc3_trb		*trb;
2292
	bool			ioc = false;
2293
	int			ret = 0;
2294

2295
	list_for_each_entry_safe(req, n, &dep->started_list, list) {
2296
		unsigned length;
2297 2298
		int chain;

2299 2300
		length = req->request.length;
		chain = req->num_pending_sgs > 0;
2301
		if (chain) {
2302
			struct scatterlist *sg = req->sg;
2303
			struct scatterlist *s;
2304
			unsigned int pending = req->num_pending_sgs;
2305
			unsigned int i;
2306

2307
			for_each_sg(sg, s, pending, i) {
2308 2309
				trb = &dep->trb_pool[dep->trb_dequeue];

2310 2311 2312
				if (trb->ctrl & DWC3_TRB_CTRL_HWO)
					break;

2313 2314 2315
				req->sg = sg_next(s);
				req->num_pending_sgs--;

2316 2317
				ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
						event, status, chain);
2318 2319
				if (ret)
					break;
2320 2321
			}
		} else {
2322
			trb = &dep->trb_pool[dep->trb_dequeue];
2323
			ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2324
					event, status, chain);
2325
		}
2326

F
Felipe Balbi 已提交
2327
		if (req->unaligned || req->zero) {
2328 2329 2330 2331
			trb = &dep->trb_pool[dep->trb_dequeue];
			ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
					event, status, false);
			req->unaligned = false;
F
Felipe Balbi 已提交
2332
			req->zero = false;
2333 2334
		}

2335
		req->request.actual = length - req->remaining;
2336

2337
		if ((req->request.actual < length) && req->num_pending_sgs)
2338
			return __dwc3_gadget_kick_transfer(dep);
2339

2340
		dwc3_gadget_giveback(dep, req, status);
2341

2342 2343 2344 2345
		if (ret) {
			if ((event->status & DEPEVT_STATUS_IOC) &&
			    (trb->ctrl & DWC3_TRB_CTRL_IOC))
				ioc = true;
2346
			break;
2347
		}
2348
	}
2349

2350 2351 2352 2353 2354 2355 2356 2357
	/*
	 * Our endpoint might get disabled by another thread during
	 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
	 * early on so DWC3_EP_BUSY flag gets cleared
	 */
	if (!dep->endpoint.desc)
		return 1;

2358
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2359 2360
			list_empty(&dep->started_list)) {
		if (list_empty(&dep->pending_list)) {
2361 2362 2363 2364 2365 2366 2367 2368
			/*
			 * If there is no entry in request list then do
			 * not issue END TRANSFER now. Just set PENDING
			 * flag, so that END TRANSFER is issued when an
			 * entry is added into request list.
			 */
			dep->flags = DWC3_EP_PENDING_REQUEST;
		} else {
2369
			dwc3_stop_active_transfer(dwc, dep->number, true);
2370 2371
			dep->flags = DWC3_EP_ENABLED;
		}
2372 2373 2374
		return 1;
	}

2375 2376 2377
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && ioc)
		return 0;

2378 2379 2380 2381
	return 1;
}

static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
2382
		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
2383 2384 2385
{
	unsigned		status = 0;
	int			clean_busy;
2386 2387 2388
	u32			is_xfer_complete;

	is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
2389 2390 2391 2392

	if (event->status & DEPEVT_STATUS_BUSERR)
		status = -ECONNRESET;

2393
	clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
2394
	if (clean_busy && (!dep->endpoint.desc || is_xfer_complete ||
2395
				usb_endpoint_xfer_isoc(dep->endpoint.desc)))
2396
		dep->flags &= ~DWC3_EP_BUSY;
2397 2398 2399 2400 2401 2402 2403 2404 2405 2406

	/*
	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
	 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
	 */
	if (dwc->revision < DWC3_REVISION_183A) {
		u32		reg;
		int		i;

		for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2407
			dep = dwc->eps[i];
2408 2409 2410 2411

			if (!(dep->flags & DWC3_EP_ENABLED))
				continue;

2412
			if (!list_empty(&dep->started_list))
2413 2414 2415 2416 2417 2418 2419 2420 2421
				return;
		}

		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
		reg |= dwc->u1u2;
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);

		dwc->u1u2 = 0;
	}
2422

2423 2424 2425 2426 2427 2428 2429 2430
	/*
	 * Our endpoint might get disabled by another thread during
	 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
	 * early on so DWC3_EP_BUSY flag gets cleared
	 */
	if (!dep->endpoint.desc)
		return;

2431 2432
	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc))
		__dwc3_gadget_kick_transfer(dep);
2433 2434 2435 2436 2437 2438 2439
}

static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
		const struct dwc3_event_depevt *event)
{
	struct dwc3_ep		*dep;
	u8			epnum = event->endpoint_number;
2440
	u8			cmd;
2441 2442 2443

	dep = dwc->eps[epnum];

2444 2445 2446 2447 2448 2449 2450 2451
	if (!(dep->flags & DWC3_EP_ENABLED)) {
		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
			return;

		/* Handle only EPCMDCMPLT when EP disabled */
		if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
			return;
	}
2452

2453 2454 2455 2456 2457 2458 2459
	if (epnum == 0 || epnum == 1) {
		dwc3_ep0_interrupt(dwc, event);
		return;
	}

	switch (event->endpoint_event) {
	case DWC3_DEPEVT_XFERCOMPLETE:
2460
		dep->resource_index = 0;
2461

2462
		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2463
			dev_err(dwc->dev, "XferComplete for Isochronous endpoint\n");
2464 2465 2466
			return;
		}

2467
		dwc3_endpoint_transfer_complete(dwc, dep, event);
2468 2469
		break;
	case DWC3_DEPEVT_XFERINPROGRESS:
2470
		dwc3_endpoint_transfer_complete(dwc, dep, event);
2471 2472
		break;
	case DWC3_DEPEVT_XFERNOTREADY:
2473
		if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
2474
			dwc3_gadget_start_isoc(dwc, dep, event);
2475 2476
		else
			__dwc3_gadget_kick_transfer(dep);
2477

2478 2479
		break;
	case DWC3_DEPEVT_STREAMEVT:
2480
		if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
2481 2482 2483 2484
			dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
					dep->name);
			return;
		}
2485 2486
		break;
	case DWC3_DEPEVT_EPCMDCMPLT:
2487 2488 2489 2490 2491 2492 2493 2494
		cmd = DEPEVT_PARAMETER_CMD(event->parameters);

		if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
			dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
			wake_up(&dep->wait_end_transfer);
		}
		break;
	case DWC3_DEPEVT_RXTXFIFOEVT:
2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507
		break;
	}
}

static void dwc3_disconnect_gadget(struct dwc3 *dwc)
{
	if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
		spin_unlock(&dwc->lock);
		dwc->gadget_driver->disconnect(&dwc->gadget);
		spin_lock(&dwc->lock);
	}
}

2508 2509
static void dwc3_suspend_gadget(struct dwc3 *dwc)
{
2510
	if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2511 2512 2513 2514 2515 2516 2517 2518
		spin_unlock(&dwc->lock);
		dwc->gadget_driver->suspend(&dwc->gadget);
		spin_lock(&dwc->lock);
	}
}

static void dwc3_resume_gadget(struct dwc3 *dwc)
{
2519
	if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2520 2521
		spin_unlock(&dwc->lock);
		dwc->gadget_driver->resume(&dwc->gadget);
2522
		spin_lock(&dwc->lock);
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533
	}
}

static void dwc3_reset_gadget(struct dwc3 *dwc)
{
	if (!dwc->gadget_driver)
		return;

	if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
		spin_unlock(&dwc->lock);
		usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2534 2535 2536 2537
		spin_lock(&dwc->lock);
	}
}

2538
static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2539 2540 2541 2542 2543 2544 2545 2546
{
	struct dwc3_ep *dep;
	struct dwc3_gadget_ep_cmd_params params;
	u32 cmd;
	int ret;

	dep = dwc->eps[epnum];

2547 2548
	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
	    !dep->resource_index)
2549 2550
		return;

2551 2552 2553 2554 2555 2556 2557 2558 2559
	/*
	 * NOTICE: We are violating what the Databook says about the
	 * EndTransfer command. Ideally we would _always_ wait for the
	 * EndTransfer Command Completion IRQ, but that's causing too
	 * much trouble synchronizing between us and gadget driver.
	 *
	 * We have discussed this with the IP Provider and it was
	 * suggested to giveback all requests here, but give HW some
	 * extra time to synchronize with the interconnect. We're using
2560
	 * an arbitrary 100us delay for that.
2561 2562 2563 2564 2565 2566 2567
	 *
	 * Note also that a similar handling was tested by Synopsys
	 * (thanks a lot Paul) and nothing bad has come out of it.
	 * In short, what we're doing is:
	 *
	 * - Issue EndTransfer WITH CMDIOC bit set
	 * - Wait 100us
2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
	 *
	 * As of IP version 3.10a of the DWC_usb3 IP, the controller
	 * supports a mode to work around the above limitation. The
	 * software can poll the CMDACT bit in the DEPCMD register
	 * after issuing a EndTransfer command. This mode is enabled
	 * by writing GUCTL2[14]. This polling is already done in the
	 * dwc3_send_gadget_ep_cmd() function so if the mode is
	 * enabled, the EndTransfer command will have completed upon
	 * returning from this function and we don't need to delay for
	 * 100us.
	 *
	 * This mode is NOT available on the DWC_usb31 IP.
2580 2581
	 */

2582
	cmd = DWC3_DEPCMD_ENDTRANSFER;
2583 2584
	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
	cmd |= DWC3_DEPCMD_CMDIOC;
2585
	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2586
	memset(&params, 0, sizeof(params));
2587
	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2588
	WARN_ON_ONCE(ret);
2589
	dep->resource_index = 0;
2590
	dep->flags &= ~DWC3_EP_BUSY;
2591

2592 2593
	if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
		dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2594
		udelay(100);
2595
	}
2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606
}

static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
{
	u32 epnum;

	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
		struct dwc3_ep *dep;
		int ret;

		dep = dwc->eps[epnum];
2607 2608
		if (!dep)
			continue;
2609 2610 2611 2612 2613 2614

		if (!(dep->flags & DWC3_EP_STALL))
			continue;

		dep->flags &= ~DWC3_EP_STALL;

2615
		ret = dwc3_send_clear_stall_ep_cmd(dep);
2616 2617 2618 2619 2620 2621
		WARN_ON_ONCE(ret);
	}
}

static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
{
2622 2623
	int			reg;

2624 2625 2626 2627 2628 2629 2630 2631 2632 2633
	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_INITU1ENA;
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	reg &= ~DWC3_DCTL_INITU2ENA;
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	dwc3_disconnect_gadget(dwc);

	dwc->gadget.speed = USB_SPEED_UNKNOWN;
2634
	dwc->setup_packet_pending = false;
2635
	usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
F
Felipe Balbi 已提交
2636 2637

	dwc->connected = false;
2638 2639 2640 2641 2642 2643
}

static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
{
	u32			reg;

F
Felipe Balbi 已提交
2644 2645
	dwc->connected = true;

2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662
	/*
	 * WORKAROUND: DWC3 revisions <1.88a have an issue which
	 * would cause a missing Disconnect Event if there's a
	 * pending Setup Packet in the FIFO.
	 *
	 * There's no suggested workaround on the official Bug
	 * report, which states that "unless the driver/application
	 * is doing any special handling of a disconnect event,
	 * there is no functional issue".
	 *
	 * Unfortunately, it turns out that we _do_ some special
	 * handling of a disconnect event, namely complete all
	 * pending transfers, notify gadget driver of the
	 * disconnection, and so on.
	 *
	 * Our suggested workaround is to follow the Disconnect
	 * Event steps here, instead, based on a setup_packet_pending
2663 2664
	 * flag. Such flag gets set whenever we have a SETUP_PENDING
	 * status for EP0 TRBs and gets cleared on XferComplete for the
2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676
	 * same endpoint.
	 *
	 * Refers to:
	 *
	 * STAR#9000466709: RTL: Device : Disconnect event not
	 * generated if setup packet pending in FIFO
	 */
	if (dwc->revision < DWC3_REVISION_188A) {
		if (dwc->setup_packet_pending)
			dwc3_gadget_disconnect_interrupt(dwc);
	}

2677
	dwc3_reset_gadget(dwc);
2678 2679 2680 2681

	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2682
	dwc->test_mode = false;
2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701
	dwc3_clear_stall_all_ep(dwc);

	/* Reset device address to zero */
	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
}

static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
{
	struct dwc3_ep		*dep;
	int			ret;
	u32			reg;
	u8			speed;

	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
	speed = reg & DWC3_DSTS_CONNECTSPD;
	dwc->speed = speed;

2702 2703 2704 2705 2706 2707 2708 2709
	/*
	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
	 * each time on Connect Done.
	 *
	 * Currently we always use the reset value. If any platform
	 * wants to set this to a different value, we need to add a
	 * setting and update GCTL.RAMCLKSEL here.
	 */
2710 2711

	switch (speed) {
2712
	case DWC3_DSTS_SUPERSPEED_PLUS:
J
John Youn 已提交
2713 2714 2715 2716
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
		dwc->gadget.ep0->maxpacket = 512;
		dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
		break;
2717
	case DWC3_DSTS_SUPERSPEED:
2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733
		/*
		 * WORKAROUND: DWC3 revisions <1.90a have an issue which
		 * would cause a missing USB3 Reset event.
		 *
		 * In such situations, we should force a USB3 Reset
		 * event by calling our dwc3_gadget_reset_interrupt()
		 * routine.
		 *
		 * Refers to:
		 *
		 * STAR#9000483510: RTL: SS : USB3 reset event may
		 * not be generated always when the link enters poll
		 */
		if (dwc->revision < DWC3_REVISION_190A)
			dwc3_gadget_reset_interrupt(dwc);

2734 2735 2736 2737
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
		dwc->gadget.ep0->maxpacket = 512;
		dwc->gadget.speed = USB_SPEED_SUPER;
		break;
2738
	case DWC3_DSTS_HIGHSPEED:
2739 2740 2741 2742
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
		dwc->gadget.ep0->maxpacket = 64;
		dwc->gadget.speed = USB_SPEED_HIGH;
		break;
2743
	case DWC3_DSTS_FULLSPEED:
2744 2745 2746 2747
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
		dwc->gadget.ep0->maxpacket = 64;
		dwc->gadget.speed = USB_SPEED_FULL;
		break;
2748
	case DWC3_DSTS_LOWSPEED:
2749 2750 2751 2752 2753 2754
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
		dwc->gadget.ep0->maxpacket = 8;
		dwc->gadget.speed = USB_SPEED_LOW;
		break;
	}

2755 2756
	/* Enable USB2 LPM Capability */

2757
	if ((dwc->revision > DWC3_REVISION_194A) &&
2758 2759
	    (speed != DWC3_DSTS_SUPERSPEED) &&
	    (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2760 2761 2762 2763 2764 2765 2766
		reg = dwc3_readl(dwc->regs, DWC3_DCFG);
		reg |= DWC3_DCFG_LPM_CAP;
		dwc3_writel(dwc->regs, DWC3_DCFG, reg);

		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
		reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);

2767
		reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2768

H
Huang Rui 已提交
2769 2770 2771 2772 2773 2774 2775 2776
		/*
		 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
		 * DCFG.LPMCap is set, core responses with an ACK and the
		 * BESL value in the LPM token is less than or equal to LPM
		 * NYET threshold.
		 */
		WARN_ONCE(dwc->revision < DWC3_REVISION_240A
				&& dwc->has_lpm_erratum,
2777
				"LPM Erratum not available on dwc3 revisions < 2.40a\n");
H
Huang Rui 已提交
2778 2779 2780 2781

		if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
			reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);

2782 2783 2784 2785
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	} else {
		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
		reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2786 2787 2788
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	}

2789
	dep = dwc->eps[0];
2790
	ret = __dwc3_gadget_ep_enable(dep, true, false);
2791 2792 2793 2794 2795 2796
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
		return;
	}

	dep = dwc->eps[1];
2797
	ret = __dwc3_gadget_ep_enable(dep, true, false);
2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
		return;
	}

	/*
	 * Configure PHY via GUSB3PIPECTLn if required.
	 *
	 * Update GTXFIFOSIZn
	 *
	 * In both cases reset values should be sufficient.
	 */
}

static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
{
	/*
	 * TODO take core out of low power mode when that's
	 * implemented.
	 */

2819 2820 2821 2822 2823
	if (dwc->gadget_driver && dwc->gadget_driver->resume) {
		spin_unlock(&dwc->lock);
		dwc->gadget_driver->resume(&dwc->gadget);
		spin_lock(&dwc->lock);
	}
2824 2825 2826 2827 2828
}

static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
		unsigned int evtinfo)
{
2829
	enum dwc3_link_state	next = evtinfo & DWC3_LINK_STATE_MASK;
2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856
	unsigned int		pwropt;

	/*
	 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
	 * Hibernation mode enabled which would show up when device detects
	 * host-initiated U3 exit.
	 *
	 * In that case, device will generate a Link State Change Interrupt
	 * from U3 to RESUME which is only necessary if Hibernation is
	 * configured in.
	 *
	 * There are no functional changes due to such spurious event and we
	 * just need to ignore it.
	 *
	 * Refers to:
	 *
	 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
	 * operational mode
	 */
	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
	if ((dwc->revision < DWC3_REVISION_250A) &&
			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
				(next == DWC3_LINK_STATE_RESUME)) {
			return;
		}
	}
2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903

	/*
	 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
	 * on the link partner, the USB session might do multiple entry/exit
	 * of low power states before a transfer takes place.
	 *
	 * Due to this problem, we might experience lower throughput. The
	 * suggested workaround is to disable DCTL[12:9] bits if we're
	 * transitioning from U1/U2 to U0 and enable those bits again
	 * after a transfer completes and there are no pending transfers
	 * on any of the enabled endpoints.
	 *
	 * This is the first half of that workaround.
	 *
	 * Refers to:
	 *
	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
	 * core send LGO_Ux entering U0
	 */
	if (dwc->revision < DWC3_REVISION_183A) {
		if (next == DWC3_LINK_STATE_U0) {
			u32	u1u2;
			u32	reg;

			switch (dwc->link_state) {
			case DWC3_LINK_STATE_U1:
			case DWC3_LINK_STATE_U2:
				reg = dwc3_readl(dwc->regs, DWC3_DCTL);
				u1u2 = reg & (DWC3_DCTL_INITU2ENA
						| DWC3_DCTL_ACCEPTU2ENA
						| DWC3_DCTL_INITU1ENA
						| DWC3_DCTL_ACCEPTU1ENA);

				if (!dwc->u1u2)
					dwc->u1u2 = reg & u1u2;

				reg &= ~u1u2;

				dwc3_writel(dwc->regs, DWC3_DCTL, reg);
				break;
			default:
				/* do nothing */
				break;
			}
		}
	}

2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920
	switch (next) {
	case DWC3_LINK_STATE_U1:
		if (dwc->speed == USB_SPEED_SUPER)
			dwc3_suspend_gadget(dwc);
		break;
	case DWC3_LINK_STATE_U2:
	case DWC3_LINK_STATE_U3:
		dwc3_suspend_gadget(dwc);
		break;
	case DWC3_LINK_STATE_RESUME:
		dwc3_resume_gadget(dwc);
		break;
	default:
		/* do nothing */
		break;
	}

2921
	dwc->link_state = next;
2922 2923
}

2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934
static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
					  unsigned int evtinfo)
{
	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;

	if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
		dwc3_suspend_gadget(dwc);

	dwc->link_state = next;
}

2935 2936 2937 2938 2939
static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
		unsigned int evtinfo)
{
	unsigned int is_ss = evtinfo & BIT(4);

F
Felipe Balbi 已提交
2940
	/*
2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958
	 * WORKAROUND: DWC3 revison 2.20a with hibernation support
	 * have a known issue which can cause USB CV TD.9.23 to fail
	 * randomly.
	 *
	 * Because of this issue, core could generate bogus hibernation
	 * events which SW needs to ignore.
	 *
	 * Refers to:
	 *
	 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
	 * Device Fallback from SuperSpeed
	 */
	if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
		return;

	/* enter hibernation here */
}

2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974
static void dwc3_gadget_interrupt(struct dwc3 *dwc,
		const struct dwc3_event_devt *event)
{
	switch (event->type) {
	case DWC3_DEVICE_EVENT_DISCONNECT:
		dwc3_gadget_disconnect_interrupt(dwc);
		break;
	case DWC3_DEVICE_EVENT_RESET:
		dwc3_gadget_reset_interrupt(dwc);
		break;
	case DWC3_DEVICE_EVENT_CONNECT_DONE:
		dwc3_gadget_conndone_interrupt(dwc);
		break;
	case DWC3_DEVICE_EVENT_WAKEUP:
		dwc3_gadget_wakeup_interrupt(dwc);
		break;
2975 2976 2977 2978 2979 2980 2981
	case DWC3_DEVICE_EVENT_HIBER_REQ:
		if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
					"unexpected hibernation event\n"))
			break;

		dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
		break;
2982 2983 2984 2985
	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
		break;
	case DWC3_DEVICE_EVENT_EOPF:
2986
		/* It changed to be suspend event for version 2.30a and above */
2987
		if (dwc->revision >= DWC3_REVISION_230A) {
2988 2989 2990 2991 2992 2993 2994 2995
			/*
			 * Ignore suspend event until the gadget enters into
			 * USB_STATE_CONFIGURED state.
			 */
			if (dwc->gadget.state >= USB_STATE_CONFIGURED)
				dwc3_gadget_suspend_interrupt(dwc,
						event->event_info);
		}
2996 2997 2998 2999 3000 3001 3002
		break;
	case DWC3_DEVICE_EVENT_SOF:
	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
	case DWC3_DEVICE_EVENT_CMD_CMPL:
	case DWC3_DEVICE_EVENT_OVERFLOW:
		break;
	default:
3003
		dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3004 3005 3006 3007 3008 3009
	}
}

static void dwc3_process_event_entry(struct dwc3 *dwc,
		const union dwc3_event *event)
{
3010
	trace_dwc3_event(event->raw, dwc);
3011

3012 3013 3014
	if (!event->type.is_devspec)
		dwc3_endpoint_interrupt(dwc, &event->depevt);
	else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3015
		dwc3_gadget_interrupt(dwc, &event->devt);
3016
	else
3017 3018 3019
		dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
}

3020
static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3021
{
3022
	struct dwc3 *dwc = evt->dwc;
3023
	irqreturn_t ret = IRQ_NONE;
3024
	int left;
3025
	u32 reg;
3026

3027
	left = evt->count;
3028

3029 3030
	if (!(evt->flags & DWC3_EVENT_PENDING))
		return IRQ_NONE;
3031

3032 3033
	while (left > 0) {
		union dwc3_event event;
3034

3035
		event.raw = *(u32 *) (evt->cache + evt->lpos);
3036

3037
		dwc3_process_event_entry(dwc, &event);
3038

3039 3040 3041 3042 3043 3044 3045 3046 3047
		/*
		 * FIXME we wrap around correctly to the next entry as
		 * almost all entries are 4 bytes in size. There is one
		 * entry which has 12 bytes which is a regular entry
		 * followed by 8 bytes data. ATM I don't know how
		 * things are organized if we get next to the a
		 * boundary so I worry about that once we try to handle
		 * that.
		 */
3048
		evt->lpos = (evt->lpos + 4) % evt->length;
3049 3050
		left -= 4;
	}
3051

3052 3053 3054
	evt->count = 0;
	evt->flags &= ~DWC3_EVENT_PENDING;
	ret = IRQ_HANDLED;
3055

3056
	/* Unmask interrupt */
3057
	reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3058
	reg &= ~DWC3_GEVNTSIZ_INTMASK;
3059
	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3060

3061 3062 3063 3064 3065
	if (dwc->imod_interval) {
		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
	}

3066 3067
	return ret;
}
3068

3069
static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3070
{
3071 3072
	struct dwc3_event_buffer *evt = _evt;
	struct dwc3 *dwc = evt->dwc;
3073
	unsigned long flags;
3074 3075
	irqreturn_t ret = IRQ_NONE;

3076
	spin_lock_irqsave(&dwc->lock, flags);
3077
	ret = dwc3_process_event_buf(evt);
3078
	spin_unlock_irqrestore(&dwc->lock, flags);
3079 3080 3081 3082

	return ret;
}

3083
static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3084
{
3085
	struct dwc3 *dwc = evt->dwc;
3086
	u32 amount;
3087
	u32 count;
3088
	u32 reg;
3089

F
Felipe Balbi 已提交
3090 3091 3092 3093 3094 3095 3096
	if (pm_runtime_suspended(dwc->dev)) {
		pm_runtime_get(dwc->dev);
		disable_irq_nosync(dwc->irq_gadget);
		dwc->pending_events = true;
		return IRQ_HANDLED;
	}

3097 3098 3099 3100 3101 3102 3103 3104 3105
	/*
	 * With PCIe legacy interrupt, test shows that top-half irq handler can
	 * be called again after HW interrupt deassertion. Check if bottom-half
	 * irq event handler completes before caching new event to prevent
	 * losing events.
	 */
	if (evt->flags & DWC3_EVENT_PENDING)
		return IRQ_HANDLED;

3106
	count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3107 3108 3109 3110
	count &= DWC3_GEVNTCOUNT_MASK;
	if (!count)
		return IRQ_NONE;

3111 3112
	evt->count = count;
	evt->flags |= DWC3_EVENT_PENDING;
3113

3114
	/* Mask interrupt */
3115
	reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3116
	reg |= DWC3_GEVNTSIZ_INTMASK;
3117
	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3118

3119 3120 3121 3122 3123 3124
	amount = min(count, evt->length - evt->lpos);
	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);

	if (amount < count)
		memcpy(evt->cache, evt->buf, count - amount);

3125 3126
	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);

3127
	return IRQ_WAKE_THREAD;
3128 3129
}

3130
static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3131
{
3132
	struct dwc3_event_buffer	*evt = _evt;
3133

3134
	return dwc3_check_event_buf(evt);
3135 3136
}

3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
static int dwc3_gadget_get_irq(struct dwc3 *dwc)
{
	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
	int irq;

	irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
	if (irq > 0)
		goto out;

	if (irq == -EPROBE_DEFER)
		goto out;

	irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
	if (irq > 0)
		goto out;

	if (irq == -EPROBE_DEFER)
		goto out;

	irq = platform_get_irq(dwc3_pdev, 0);
	if (irq > 0)
		goto out;

	if (irq != -EPROBE_DEFER)
		dev_err(dwc->dev, "missing peripheral IRQ\n");

	if (!irq)
		irq = -EINVAL;

out:
	return irq;
}

3170
/**
F
Felipe Balbi 已提交
3171
 * dwc3_gadget_init - initializes gadget related registers
3172
 * @dwc: pointer to our controller context structure
3173 3174 3175
 *
 * Returns 0 on success otherwise negative errno.
 */
B
Bill Pemberton 已提交
3176
int dwc3_gadget_init(struct dwc3 *dwc)
3177
{
3178 3179
	int ret;
	int irq;
3180

3181 3182 3183 3184
	irq = dwc3_gadget_get_irq(dwc);
	if (irq < 0) {
		ret = irq;
		goto err0;
3185 3186 3187
	}

	dwc->irq_gadget = irq;
3188

3189 3190 3191
	dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
					  sizeof(*dwc->ep0_trb) * 2,
					  &dwc->ep0_trb_addr, GFP_KERNEL);
3192 3193 3194
	if (!dwc->ep0_trb) {
		dev_err(dwc->dev, "failed to allocate ep0 trb\n");
		ret = -ENOMEM;
3195
		goto err0;
3196 3197
	}

3198
	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3199 3200
	if (!dwc->setup_buf) {
		ret = -ENOMEM;
3201
		goto err1;
3202 3203
	}

3204 3205 3206 3207
	dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
			&dwc->bounce_addr, GFP_KERNEL);
	if (!dwc->bounce) {
		ret = -ENOMEM;
F
Felipe Balbi 已提交
3208
		goto err2;
3209 3210
	}

3211 3212
	init_completion(&dwc->ep0_in_setup);

3213 3214
	dwc->gadget.ops			= &dwc3_gadget_ops;
	dwc->gadget.speed		= USB_SPEED_UNKNOWN;
3215
	dwc->gadget.sg_supported	= true;
3216
	dwc->gadget.name		= "dwc3-gadget";
3217
	dwc->gadget.is_otg		= dwc->dr_mode == USB_DR_MODE_OTG;
3218

3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
	/*
	 * FIXME We might be setting max_speed to <SUPER, however versions
	 * <2.20a of dwc3 have an issue with metastability (documented
	 * elsewhere in this driver) which tells us we can't set max speed to
	 * anything lower than SUPER.
	 *
	 * Because gadget.max_speed is only used by composite.c and function
	 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
	 * to happen so we avoid sending SuperSpeed Capability descriptor
	 * together with our BOS descriptor as that could confuse host into
	 * thinking we can handle super speed.
	 *
	 * Note that, in fact, we won't even support GetBOS requests when speed
	 * is less than super speed because we don't have means, yet, to tell
	 * composite.c that we are USB 2.0 + LPM ECN.
	 */
	if (dwc->revision < DWC3_REVISION_220A)
3236
		dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3237 3238 3239 3240
				dwc->revision);

	dwc->gadget.max_speed		= dwc->maximum_speed;

3241 3242 3243 3244 3245
	/*
	 * REVISIT: Here we should clear all pending IRQs to be
	 * sure we're starting from a well known location.
	 */

3246
	ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3247
	if (ret)
F
Felipe Balbi 已提交
3248
		goto err3;
3249 3250 3251 3252

	ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
	if (ret) {
		dev_err(dwc->dev, "failed to register udc\n");
F
Felipe Balbi 已提交
3253
		goto err4;
3254 3255 3256 3257
	}

	return 0;

3258
err4:
F
Felipe Balbi 已提交
3259
	dwc3_gadget_free_endpoints(dwc);
3260

3261
err3:
F
Felipe Balbi 已提交
3262 3263
	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
			dwc->bounce_addr);
3264

3265
err2:
3266
	kfree(dwc->setup_buf);
3267

3268
err1:
3269
	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3270 3271 3272 3273 3274 3275
			dwc->ep0_trb, dwc->ep0_trb_addr);

err0:
	return ret;
}

3276 3277
/* -------------------------------------------------------------------------- */

3278 3279 3280 3281
void dwc3_gadget_exit(struct dwc3 *dwc)
{
	usb_del_gadget_udc(&dwc->gadget);
	dwc3_gadget_free_endpoints(dwc);
3282
	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
F
Felipe Balbi 已提交
3283
			  dwc->bounce_addr);
3284
	kfree(dwc->setup_buf);
3285
	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
F
Felipe Balbi 已提交
3286
			  dwc->ep0_trb, dwc->ep0_trb_addr);
3287
}
3288

3289
int dwc3_gadget_suspend(struct dwc3 *dwc)
3290
{
3291 3292 3293
	if (!dwc->gadget_driver)
		return 0;

3294
	dwc3_gadget_run_stop(dwc, false, false);
3295 3296
	dwc3_disconnect_gadget(dwc);
	__dwc3_gadget_stop(dwc);
3297 3298 3299 3300 3301 3302 3303 3304

	return 0;
}

int dwc3_gadget_resume(struct dwc3 *dwc)
{
	int			ret;

3305 3306 3307
	if (!dwc->gadget_driver)
		return 0;

3308 3309
	ret = __dwc3_gadget_start(dwc);
	if (ret < 0)
3310 3311
		goto err0;

3312 3313
	ret = dwc3_gadget_run_stop(dwc, true, false);
	if (ret < 0)
3314 3315 3316 3317 3318
		goto err1;

	return 0;

err1:
3319
	__dwc3_gadget_stop(dwc);
3320 3321 3322 3323

err0:
	return ret;
}
F
Felipe Balbi 已提交
3324 3325 3326 3327 3328 3329 3330 3331 3332

void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
{
	if (dwc->pending_events) {
		dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
		dwc->pending_events = false;
		enable_irq(dwc->irq_gadget);
	}
}