blob: e47cba55c431ec73acfe488b539943afb75e74eb [file] [log] [blame]
Felipe Balbi72246da2011-08-19 18:10:58 +03001/**
2 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
3 *
4 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
Felipe Balbi72246da2011-08-19 18:10:58 +03005 *
6 * Authors: Felipe Balbi <balbi@ti.com>,
7 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
8 *
Felipe Balbi5945f782013-06-30 14:15:11 +03009 * This program is free software: you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 of
11 * the License as published by the Free Software Foundation.
Felipe Balbi72246da2011-08-19 18:10:58 +030012 *
Felipe Balbi5945f782013-06-30 14:15:11 +030013 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
Felipe Balbi72246da2011-08-19 18:10:58 +030017 */
18
19#include <linux/kernel.h>
20#include <linux/delay.h>
21#include <linux/slab.h>
22#include <linux/spinlock.h>
23#include <linux/platform_device.h>
24#include <linux/pm_runtime.h>
25#include <linux/interrupt.h>
26#include <linux/io.h>
27#include <linux/list.h>
28#include <linux/dma-mapping.h>
29
30#include <linux/usb/ch9.h>
31#include <linux/usb/gadget.h>
32
Felipe Balbi80977dc2014-08-19 16:37:22 -050033#include "debug.h"
Felipe Balbi72246da2011-08-19 18:10:58 +030034#include "core.h"
35#include "gadget.h"
36#include "io.h"
37
Felipe Balbi04a9bfc2012-01-02 18:25:43 +020038/**
39 * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
40 * @dwc: pointer to our context structure
41 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
42 *
43 * Caller should take care of locking. This function will
44 * return 0 on success or -EINVAL if wrong Test Selector
45 * is passed
46 */
47int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
48{
49 u32 reg;
50
51 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
52 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
53
54 switch (mode) {
55 case TEST_J:
56 case TEST_K:
57 case TEST_SE0_NAK:
58 case TEST_PACKET:
59 case TEST_FORCE_EN:
60 reg |= mode << 1;
61 break;
62 default:
63 return -EINVAL;
64 }
65
66 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
67
68 return 0;
69}
70
Felipe Balbi8598bde2012-01-02 18:55:57 +020071/**
Paul Zimmerman911f1f82012-04-27 13:35:15 +030072 * dwc3_gadget_get_link_state - Gets current state of USB Link
73 * @dwc: pointer to our context structure
74 *
75 * Caller should take care of locking. This function will
76 * return the link state on success (>= 0) or -ETIMEDOUT.
77 */
78int dwc3_gadget_get_link_state(struct dwc3 *dwc)
79{
80 u32 reg;
81
82 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
83
84 return DWC3_DSTS_USBLNKST(reg);
85}
86
87/**
Felipe Balbi8598bde2012-01-02 18:55:57 +020088 * dwc3_gadget_set_link_state - Sets USB Link to a particular State
89 * @dwc: pointer to our context structure
90 * @state: the state to put link into
91 *
92 * Caller should take care of locking. This function will
Paul Zimmermanaee63e32012-02-24 17:32:15 -080093 * return 0 on success or -ETIMEDOUT.
Felipe Balbi8598bde2012-01-02 18:55:57 +020094 */
95int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
96{
Paul Zimmermanaee63e32012-02-24 17:32:15 -080097 int retries = 10000;
Felipe Balbi8598bde2012-01-02 18:55:57 +020098 u32 reg;
99
Paul Zimmerman802fde92012-04-27 13:10:52 +0300100 /*
101 * Wait until device controller is ready. Only applies to 1.94a and
102 * later RTL.
103 */
104 if (dwc->revision >= DWC3_REVISION_194A) {
105 while (--retries) {
106 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
107 if (reg & DWC3_DSTS_DCNRD)
108 udelay(5);
109 else
110 break;
111 }
112
113 if (retries <= 0)
114 return -ETIMEDOUT;
115 }
116
Felipe Balbi8598bde2012-01-02 18:55:57 +0200117 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
118 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
119
120 /* set requested state */
121 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
122 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
123
Paul Zimmerman802fde92012-04-27 13:10:52 +0300124 /*
125 * The following code is racy when called from dwc3_gadget_wakeup,
126 * and is not needed, at least on newer versions
127 */
128 if (dwc->revision >= DWC3_REVISION_194A)
129 return 0;
130
Felipe Balbi8598bde2012-01-02 18:55:57 +0200131 /* wait for a change in DSTS */
Paul Zimmermanaed430e2012-04-27 12:52:01 +0300132 retries = 10000;
Felipe Balbi8598bde2012-01-02 18:55:57 +0200133 while (--retries) {
134 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
135
Felipe Balbi8598bde2012-01-02 18:55:57 +0200136 if (DWC3_DSTS_USBLNKST(reg) == state)
137 return 0;
138
Paul Zimmermanaee63e32012-02-24 17:32:15 -0800139 udelay(5);
Felipe Balbi8598bde2012-01-02 18:55:57 +0200140 }
141
Felipe Balbi8598bde2012-01-02 18:55:57 +0200142 return -ETIMEDOUT;
143}
144
John Youndca01192016-05-19 17:26:05 -0700145/**
146 * dwc3_ep_inc_trb() - Increment a TRB index.
147 * @index - Pointer to the TRB index to increment.
148 *
149 * The index should never point to the link TRB. After incrementing,
150 * if it is point to the link TRB, wrap around to the beginning. The
151 * link TRB is always at the last TRB entry.
152 */
153static void dwc3_ep_inc_trb(u8 *index)
154{
155 (*index)++;
156 if (*index == (DWC3_TRB_NUM - 1))
157 *index = 0;
158}
159
Felipe Balbief966b92016-04-05 13:09:51 +0300160static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
Felipe Balbi457e84b2012-01-18 18:04:09 +0200161{
John Youndca01192016-05-19 17:26:05 -0700162 dwc3_ep_inc_trb(&dep->trb_enqueue);
Felipe Balbief966b92016-04-05 13:09:51 +0300163}
Felipe Balbi457e84b2012-01-18 18:04:09 +0200164
Felipe Balbief966b92016-04-05 13:09:51 +0300165static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
166{
John Youndca01192016-05-19 17:26:05 -0700167 dwc3_ep_inc_trb(&dep->trb_dequeue);
Felipe Balbi457e84b2012-01-18 18:04:09 +0200168}
169
Felipe Balbi72246da2011-08-19 18:10:58 +0300170void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
171 int status)
172{
173 struct dwc3 *dwc = dep->dwc;
174
Felipe Balbi737f1ae2016-08-11 12:24:27 +0300175 req->started = false;
Felipe Balbi72246da2011-08-19 18:10:58 +0300176 list_del(&req->list);
Felipe Balbieeb720f2011-11-28 12:46:59 +0200177 req->trb = NULL;
Felipe Balbie62c5bc52016-10-25 13:47:21 +0300178 req->remaining = 0;
Felipe Balbi72246da2011-08-19 18:10:58 +0300179
180 if (req->request.status == -EINPROGRESS)
181 req->request.status = status;
182
Pratyush Anand0416e492012-08-10 13:42:16 +0530183 if (dwc->ep0_bounced && dep->number == 0)
184 dwc->ep0_bounced = false;
185 else
186 usb_gadget_unmap_request(&dwc->gadget, &req->request,
187 req->direction);
Felipe Balbi72246da2011-08-19 18:10:58 +0300188
Felipe Balbi2c4cbe6e52014-04-30 17:45:10 -0500189 trace_dwc3_gadget_giveback(req);
Felipe Balbi72246da2011-08-19 18:10:58 +0300190
191 spin_unlock(&dwc->lock);
Michal Sojka304f7e52014-09-24 22:43:19 +0200192 usb_gadget_giveback_request(&dep->endpoint, &req->request);
Felipe Balbi72246da2011-08-19 18:10:58 +0300193 spin_lock(&dwc->lock);
Felipe Balbifc8bb912016-05-16 13:14:48 +0300194
195 if (dep->number > 1)
196 pm_runtime_put(dwc->dev);
Felipe Balbi72246da2011-08-19 18:10:58 +0300197}
198
Felipe Balbi3ece0ec2014-09-05 09:47:44 -0500199int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
Felipe Balbib09bb642012-04-24 16:19:11 +0300200{
201 u32 timeout = 500;
Felipe Balbi71f7e702016-05-23 14:16:19 +0300202 int status = 0;
Felipe Balbi0fe886c2016-05-23 14:06:07 +0300203 int ret = 0;
Felipe Balbib09bb642012-04-24 16:19:11 +0300204 u32 reg;
205
206 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
207 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
208
209 do {
210 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
211 if (!(reg & DWC3_DGCMD_CMDACT)) {
Felipe Balbi71f7e702016-05-23 14:16:19 +0300212 status = DWC3_DGCMD_STATUS(reg);
213 if (status)
Felipe Balbi0fe886c2016-05-23 14:06:07 +0300214 ret = -EINVAL;
215 break;
Felipe Balbib09bb642012-04-24 16:19:11 +0300216 }
Felipe Balbi0fe886c2016-05-23 14:06:07 +0300217 } while (timeout--);
218
219 if (!timeout) {
Felipe Balbi0fe886c2016-05-23 14:06:07 +0300220 ret = -ETIMEDOUT;
Felipe Balbi71f7e702016-05-23 14:16:19 +0300221 status = -ETIMEDOUT;
Felipe Balbi0fe886c2016-05-23 14:06:07 +0300222 }
223
Felipe Balbi71f7e702016-05-23 14:16:19 +0300224 trace_dwc3_gadget_generic_cmd(cmd, param, status);
225
Felipe Balbi0fe886c2016-05-23 14:06:07 +0300226 return ret;
Felipe Balbib09bb642012-04-24 16:19:11 +0300227}
228
Felipe Balbic36d8e92016-04-04 12:46:33 +0300229static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
230
Felipe Balbi2cd47182016-04-12 16:42:43 +0300231int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
232 struct dwc3_gadget_ep_cmd_params *params)
Felipe Balbi72246da2011-08-19 18:10:58 +0300233{
Felipe Balbi8897a762016-09-22 10:56:08 +0300234 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
Felipe Balbi2cd47182016-04-12 16:42:43 +0300235 struct dwc3 *dwc = dep->dwc;
Sebastian Andrzej Siewior61d58242011-08-29 16:46:38 +0200236 u32 timeout = 500;
Felipe Balbi72246da2011-08-19 18:10:58 +0300237 u32 reg;
238
Felipe Balbi0933df12016-05-23 14:02:33 +0300239 int cmd_status = 0;
Felipe Balbi2b0f11d2016-04-04 09:19:17 +0300240 int susphy = false;
Felipe Balbic0ca3242016-04-04 09:11:51 +0300241 int ret = -EINVAL;
Felipe Balbi72246da2011-08-19 18:10:58 +0300242
Felipe Balbi2b0f11d2016-04-04 09:19:17 +0300243 /*
244 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
245 * we're issuing an endpoint command, we must check if
246 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
247 *
248 * We will also set SUSPHY bit to what it was before returning as stated
249 * by the same section on Synopsys databook.
250 */
Felipe Balbiab2a92e2016-05-17 14:55:58 +0300251 if (dwc->gadget.speed <= USB_SPEED_HIGH) {
252 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
253 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
254 susphy = true;
255 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
256 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
257 }
Felipe Balbi2b0f11d2016-04-04 09:19:17 +0300258 }
259
Felipe Balbi59999142016-09-22 12:25:28 +0300260 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
Felipe Balbic36d8e92016-04-04 12:46:33 +0300261 int needs_wakeup;
262
263 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
264 dwc->link_state == DWC3_LINK_STATE_U2 ||
265 dwc->link_state == DWC3_LINK_STATE_U3);
266
267 if (unlikely(needs_wakeup)) {
268 ret = __dwc3_gadget_wakeup(dwc);
269 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
270 ret);
271 }
272 }
273
Felipe Balbi2eb88012016-04-12 16:53:39 +0300274 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
275 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
276 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
Felipe Balbi72246da2011-08-19 18:10:58 +0300277
Felipe Balbi8897a762016-09-22 10:56:08 +0300278 /*
279 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
280 * not relying on XferNotReady, we can make use of a special "No
281 * Response Update Transfer" command where we should clear both CmdAct
282 * and CmdIOC bits.
283 *
284 * With this, we don't need to wait for command completion and can
285 * straight away issue further commands to the endpoint.
286 *
287 * NOTICE: We're making an assumption that control endpoints will never
288 * make use of Update Transfer command. This is a safe assumption
289 * because we can never have more than one request at a time with
290 * Control Endpoints. If anybody changes that assumption, this chunk
291 * needs to be updated accordingly.
292 */
293 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
294 !usb_endpoint_xfer_isoc(desc))
295 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
296 else
297 cmd |= DWC3_DEPCMD_CMDACT;
298
299 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
Felipe Balbi72246da2011-08-19 18:10:58 +0300300 do {
Felipe Balbi2eb88012016-04-12 16:53:39 +0300301 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
Felipe Balbi72246da2011-08-19 18:10:58 +0300302 if (!(reg & DWC3_DEPCMD_CMDACT)) {
Felipe Balbi0933df12016-05-23 14:02:33 +0300303 cmd_status = DWC3_DEPCMD_STATUS(reg);
Konrad Leszczynski7b9cc7a2016-02-12 15:21:46 +0000304
Konrad Leszczynski7b9cc7a2016-02-12 15:21:46 +0000305 switch (cmd_status) {
306 case 0:
307 ret = 0;
Felipe Balbic0ca3242016-04-04 09:11:51 +0300308 break;
Konrad Leszczynski7b9cc7a2016-02-12 15:21:46 +0000309 case DEPEVT_TRANSFER_NO_RESOURCE:
Konrad Leszczynski7b9cc7a2016-02-12 15:21:46 +0000310 ret = -EINVAL;
311 break;
312 case DEPEVT_TRANSFER_BUS_EXPIRY:
313 /*
314 * SW issues START TRANSFER command to
315 * isochronous ep with future frame interval. If
316 * future interval time has already passed when
317 * core receives the command, it will respond
318 * with an error status of 'Bus Expiry'.
319 *
320 * Instead of always returning -EINVAL, let's
321 * give a hint to the gadget driver that this is
322 * the case by returning -EAGAIN.
323 */
Konrad Leszczynski7b9cc7a2016-02-12 15:21:46 +0000324 ret = -EAGAIN;
325 break;
326 default:
327 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
328 }
329
Felipe Balbic0ca3242016-04-04 09:11:51 +0300330 break;
Felipe Balbi72246da2011-08-19 18:10:58 +0300331 }
Felipe Balbif6bb2252016-05-23 13:53:34 +0300332 } while (--timeout);
Felipe Balbi72246da2011-08-19 18:10:58 +0300333
Felipe Balbif6bb2252016-05-23 13:53:34 +0300334 if (timeout == 0) {
Felipe Balbif6bb2252016-05-23 13:53:34 +0300335 ret = -ETIMEDOUT;
Felipe Balbi0933df12016-05-23 14:02:33 +0300336 cmd_status = -ETIMEDOUT;
Felipe Balbif6bb2252016-05-23 13:53:34 +0300337 }
Felipe Balbic0ca3242016-04-04 09:11:51 +0300338
Felipe Balbi0933df12016-05-23 14:02:33 +0300339 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
340
Felipe Balbi6cb2e4e2016-10-21 13:07:09 +0300341 if (ret == 0) {
342 switch (DWC3_DEPCMD_CMD(cmd)) {
343 case DWC3_DEPCMD_STARTTRANSFER:
344 dep->flags |= DWC3_EP_TRANSFER_STARTED;
345 break;
346 case DWC3_DEPCMD_ENDTRANSFER:
347 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
348 break;
349 default:
350 /* nothing */
351 break;
352 }
353 }
354
Felipe Balbi2b0f11d2016-04-04 09:19:17 +0300355 if (unlikely(susphy)) {
356 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
357 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
358 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
359 }
360
Felipe Balbic0ca3242016-04-04 09:11:51 +0300361 return ret;
Felipe Balbi72246da2011-08-19 18:10:58 +0300362}
363
John Youn50c763f2016-05-31 17:49:56 -0700364static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
365{
366 struct dwc3 *dwc = dep->dwc;
367 struct dwc3_gadget_ep_cmd_params params;
368 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
369
370 /*
371 * As of core revision 2.60a the recommended programming model
372 * is to set the ClearPendIN bit when issuing a Clear Stall EP
373 * command for IN endpoints. This is to prevent an issue where
374 * some (non-compliant) hosts may not send ACK TPs for pending
375 * IN transfers due to a mishandled error condition. Synopsys
376 * STAR 9000614252.
377 */
Lu Baolu5e6c88d2016-09-09 12:51:27 +0800378 if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
379 (dwc->gadget.speed >= USB_SPEED_SUPER))
John Youn50c763f2016-05-31 17:49:56 -0700380 cmd |= DWC3_DEPCMD_CLEARPENDIN;
381
382 memset(&params, 0, sizeof(params));
383
Felipe Balbi2cd47182016-04-12 16:42:43 +0300384 return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
John Youn50c763f2016-05-31 17:49:56 -0700385}
386
Felipe Balbi72246da2011-08-19 18:10:58 +0300387static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
Felipe Balbif6bafc62012-02-06 11:04:53 +0200388 struct dwc3_trb *trb)
Felipe Balbi72246da2011-08-19 18:10:58 +0300389{
Paul Zimmermanc439ef82011-09-30 10:58:45 +0300390 u32 offset = (char *) trb - (char *) dep->trb_pool;
Felipe Balbi72246da2011-08-19 18:10:58 +0300391
392 return dep->trb_pool_dma + offset;
393}
394
395static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
396{
397 struct dwc3 *dwc = dep->dwc;
398
399 if (dep->trb_pool)
400 return 0;
401
Felipe Balbi72246da2011-08-19 18:10:58 +0300402 dep->trb_pool = dma_alloc_coherent(dwc->dev,
403 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
404 &dep->trb_pool_dma, GFP_KERNEL);
405 if (!dep->trb_pool) {
406 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
407 dep->name);
408 return -ENOMEM;
409 }
410
411 return 0;
412}
413
414static void dwc3_free_trb_pool(struct dwc3_ep *dep)
415{
416 struct dwc3 *dwc = dep->dwc;
417
418 dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
419 dep->trb_pool, dep->trb_pool_dma);
420
421 dep->trb_pool = NULL;
422 dep->trb_pool_dma = 0;
423}
424
John Younc4509602016-02-16 20:10:53 -0800425static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
426
427/**
428 * dwc3_gadget_start_config - Configure EP resources
429 * @dwc: pointer to our controller context structure
430 * @dep: endpoint that is being enabled
431 *
432 * The assignment of transfer resources cannot perfectly follow the
433 * data book due to the fact that the controller driver does not have
434 * all knowledge of the configuration in advance. It is given this
435 * information piecemeal by the composite gadget framework after every
436 * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
437 * programming model in this scenario can cause errors. For two
438 * reasons:
439 *
440 * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
441 * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
442 * multiple interfaces.
443 *
444 * 2) The databook does not mention doing more DEPXFERCFG for new
445 * endpoint on alt setting (8.1.6).
446 *
447 * The following simplified method is used instead:
448 *
449 * All hardware endpoints can be assigned a transfer resource and this
450 * setting will stay persistent until either a core reset or
451 * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
452 * do DEPXFERCFG for every hardware endpoint as well. We are
453 * guaranteed that there are as many transfer resources as endpoints.
454 *
455 * This function is called for each endpoint when it is being enabled
456 * but is triggered only when called for EP0-out, which always happens
457 * first, and which should only happen in one of the above conditions.
458 */
Felipe Balbi72246da2011-08-19 18:10:58 +0300459static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
460{
461 struct dwc3_gadget_ep_cmd_params params;
462 u32 cmd;
John Younc4509602016-02-16 20:10:53 -0800463 int i;
464 int ret;
465
466 if (dep->number)
467 return 0;
Felipe Balbi72246da2011-08-19 18:10:58 +0300468
469 memset(&params, 0x00, sizeof(params));
John Younc4509602016-02-16 20:10:53 -0800470 cmd = DWC3_DEPCMD_DEPSTARTCFG;
Felipe Balbi72246da2011-08-19 18:10:58 +0300471
Felipe Balbi2cd47182016-04-12 16:42:43 +0300472 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
John Younc4509602016-02-16 20:10:53 -0800473 if (ret)
474 return ret;
Felipe Balbi72246da2011-08-19 18:10:58 +0300475
John Younc4509602016-02-16 20:10:53 -0800476 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
477 struct dwc3_ep *dep = dwc->eps[i];
478
479 if (!dep)
480 continue;
481
482 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
483 if (ret)
484 return ret;
Felipe Balbi72246da2011-08-19 18:10:58 +0300485 }
486
487 return 0;
488}
489
490static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
Felipe Balbic90bfae2011-11-29 13:11:21 +0200491 const struct usb_endpoint_descriptor *desc,
Felipe Balbi4b345c92012-07-16 14:08:16 +0300492 const struct usb_ss_ep_comp_descriptor *comp_desc,
Felipe Balbi21e64bf2016-06-02 12:37:31 +0300493 bool modify, bool restore)
Felipe Balbi72246da2011-08-19 18:10:58 +0300494{
495 struct dwc3_gadget_ep_cmd_params params;
496
Felipe Balbi21e64bf2016-06-02 12:37:31 +0300497 if (dev_WARN_ONCE(dwc->dev, modify && restore,
498 "Can't modify and restore\n"))
499 return -EINVAL;
500
Felipe Balbi72246da2011-08-19 18:10:58 +0300501 memset(&params, 0x00, sizeof(params));
502
Felipe Balbidc1c70a2011-09-30 10:58:51 +0300503 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
Chanho Parkd2e9a132012-08-31 16:54:07 +0900504 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
505
506 /* Burst size is only needed in SuperSpeed mode */
John Younee5cd412016-02-05 17:08:45 -0800507 if (dwc->gadget.speed >= USB_SPEED_SUPER) {
Felipe Balbi676e3492016-04-26 10:49:07 +0300508 u32 burst = dep->endpoint.maxburst;
Felipe Balbi676e3492016-04-26 10:49:07 +0300509 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
Chanho Parkd2e9a132012-08-31 16:54:07 +0900510 }
Felipe Balbi72246da2011-08-19 18:10:58 +0300511
Felipe Balbi21e64bf2016-06-02 12:37:31 +0300512 if (modify) {
513 params.param0 |= DWC3_DEPCFG_ACTION_MODIFY;
514 } else if (restore) {
Paul Zimmerman265b70a2013-12-19 12:38:49 -0600515 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
516 params.param2 |= dep->saved_state;
Felipe Balbi21e64bf2016-06-02 12:37:31 +0300517 } else {
518 params.param0 |= DWC3_DEPCFG_ACTION_INIT;
Paul Zimmerman265b70a2013-12-19 12:38:49 -0600519 }
520
Felipe Balbi4bc48c92016-08-10 16:04:33 +0300521 if (usb_endpoint_xfer_control(desc))
522 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
Felipe Balbi13fa2e62016-05-30 13:40:00 +0300523
524 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
525 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
Felipe Balbi72246da2011-08-19 18:10:58 +0300526
Felipe Balbi18b7ede2012-01-02 13:35:41 +0200527 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
Felipe Balbidc1c70a2011-09-30 10:58:51 +0300528 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
529 | DWC3_DEPCFG_STREAM_EVENT_EN;
Felipe Balbi879631a2011-09-30 10:58:47 +0300530 dep->stream_capable = true;
531 }
532
Felipe Balbi0b93a4c2014-09-04 10:28:10 -0500533 if (!usb_endpoint_xfer_control(desc))
Felipe Balbidc1c70a2011-09-30 10:58:51 +0300534 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
Felipe Balbi72246da2011-08-19 18:10:58 +0300535
536 /*
537 * We are doing 1:1 mapping for endpoints, meaning
538 * Physical Endpoints 2 maps to Logical Endpoint 2 and
539 * so on. We consider the direction bit as part of the physical
540 * endpoint number. So USB endpoint 0x81 is 0x03.
541 */
Felipe Balbidc1c70a2011-09-30 10:58:51 +0300542 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
Felipe Balbi72246da2011-08-19 18:10:58 +0300543
544 /*
545 * We must use the lower 16 TX FIFOs even though
546 * HW might have more
547 */
548 if (dep->direction)
Felipe Balbidc1c70a2011-09-30 10:58:51 +0300549 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
Felipe Balbi72246da2011-08-19 18:10:58 +0300550
551 if (desc->bInterval) {
Felipe Balbidc1c70a2011-09-30 10:58:51 +0300552 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
Felipe Balbi72246da2011-08-19 18:10:58 +0300553 dep->interval = 1 << (desc->bInterval - 1);
554 }
555
Felipe Balbi2cd47182016-04-12 16:42:43 +0300556 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
Felipe Balbi72246da2011-08-19 18:10:58 +0300557}
558
559static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
560{
561 struct dwc3_gadget_ep_cmd_params params;
562
563 memset(&params, 0x00, sizeof(params));
564
Felipe Balbidc1c70a2011-09-30 10:58:51 +0300565 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
Felipe Balbi72246da2011-08-19 18:10:58 +0300566
Felipe Balbi2cd47182016-04-12 16:42:43 +0300567 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
568 &params);
Felipe Balbi72246da2011-08-19 18:10:58 +0300569}
570
571/**
572 * __dwc3_gadget_ep_enable - Initializes a HW endpoint
573 * @dep: endpoint to be initialized
574 * @desc: USB Endpoint Descriptor
575 *
576 * Caller should take care of locking
577 */
578static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
Felipe Balbic90bfae2011-11-29 13:11:21 +0200579 const struct usb_endpoint_descriptor *desc,
Felipe Balbi4b345c92012-07-16 14:08:16 +0300580 const struct usb_ss_ep_comp_descriptor *comp_desc,
Felipe Balbi21e64bf2016-06-02 12:37:31 +0300581 bool modify, bool restore)
Felipe Balbi72246da2011-08-19 18:10:58 +0300582{
583 struct dwc3 *dwc = dep->dwc;
584 u32 reg;
Andy Shevchenkob09e99e2014-05-15 15:53:32 +0300585 int ret;
Felipe Balbi72246da2011-08-19 18:10:58 +0300586
587 if (!(dep->flags & DWC3_EP_ENABLED)) {
588 ret = dwc3_gadget_start_config(dwc, dep);
589 if (ret)
590 return ret;
591 }
592
Felipe Balbi21e64bf2016-06-02 12:37:31 +0300593 ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, modify,
Paul Zimmerman265b70a2013-12-19 12:38:49 -0600594 restore);
Felipe Balbi72246da2011-08-19 18:10:58 +0300595 if (ret)
596 return ret;
597
598 if (!(dep->flags & DWC3_EP_ENABLED)) {
Felipe Balbif6bafc62012-02-06 11:04:53 +0200599 struct dwc3_trb *trb_st_hw;
600 struct dwc3_trb *trb_link;
Felipe Balbi72246da2011-08-19 18:10:58 +0300601
Ido Shayevitz16e78db2012-03-12 20:25:24 +0200602 dep->endpoint.desc = desc;
Felipe Balbic90bfae2011-11-29 13:11:21 +0200603 dep->comp_desc = comp_desc;
Felipe Balbi72246da2011-08-19 18:10:58 +0300604 dep->type = usb_endpoint_type(desc);
605 dep->flags |= DWC3_EP_ENABLED;
Baolin Wang76a638f2016-10-31 19:38:36 +0800606 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
Felipe Balbi72246da2011-08-19 18:10:58 +0300607
608 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
609 reg |= DWC3_DALEPENA_EP(dep->number);
610 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
611
Baolin Wang76a638f2016-10-31 19:38:36 +0800612 init_waitqueue_head(&dep->wait_end_transfer);
613
Felipe Balbi36b68aa2016-04-05 13:24:36 +0300614 if (usb_endpoint_xfer_control(desc))
Felipe Balbi2870e502016-11-03 13:53:29 +0200615 goto out;
Felipe Balbi72246da2011-08-19 18:10:58 +0300616
John Youn0d257442016-05-19 17:26:08 -0700617 /* Initialize the TRB ring */
618 dep->trb_dequeue = 0;
619 dep->trb_enqueue = 0;
620 memset(dep->trb_pool, 0,
621 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
622
Felipe Balbi36b68aa2016-04-05 13:24:36 +0300623 /* Link TRB. The HWO bit is never reset */
Felipe Balbi72246da2011-08-19 18:10:58 +0300624 trb_st_hw = &dep->trb_pool[0];
625
Felipe Balbif6bafc62012-02-06 11:04:53 +0200626 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
Felipe Balbif6bafc62012-02-06 11:04:53 +0200627 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
628 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
629 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
630 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
Felipe Balbi72246da2011-08-19 18:10:58 +0300631 }
632
Felipe Balbia97ea992016-09-29 16:28:56 +0300633 /*
634 * Issue StartTransfer here with no-op TRB so we can always rely on No
635 * Response Update Transfer command.
636 */
637 if (usb_endpoint_xfer_bulk(desc)) {
638 struct dwc3_gadget_ep_cmd_params params;
639 struct dwc3_trb *trb;
640 dma_addr_t trb_dma;
641 u32 cmd;
642
643 memset(&params, 0, sizeof(params));
644 trb = &dep->trb_pool[0];
645 trb_dma = dwc3_trb_dma_offset(dep, trb);
646
647 params.param0 = upper_32_bits(trb_dma);
648 params.param1 = lower_32_bits(trb_dma);
649
650 cmd = DWC3_DEPCMD_STARTTRANSFER;
651
652 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
653 if (ret < 0)
654 return ret;
655
656 dep->flags |= DWC3_EP_BUSY;
657
658 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
659 WARN_ON_ONCE(!dep->resource_index);
660 }
661
Felipe Balbi2870e502016-11-03 13:53:29 +0200662
663out:
664 trace_dwc3_gadget_ep_enable(dep);
665
Felipe Balbi72246da2011-08-19 18:10:58 +0300666 return 0;
667}
668
Paul Zimmermanb992e682012-04-27 14:17:35 +0300669static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
Sebastian Andrzej Siewior624407f2011-08-29 13:56:37 +0200670static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
Felipe Balbi72246da2011-08-19 18:10:58 +0300671{
672 struct dwc3_request *req;
673
Felipe Balbi0e146022016-06-21 10:32:02 +0300674 dwc3_stop_active_transfer(dwc, dep->number, true);
Felipe Balbi69450c42016-05-30 13:37:02 +0300675
Felipe Balbi0e146022016-06-21 10:32:02 +0300676 /* - giveback all requests to gadget driver */
677 while (!list_empty(&dep->started_list)) {
678 req = next_request(&dep->started_list);
Sebastian Andrzej Siewior624407f2011-08-29 13:56:37 +0200679
Felipe Balbi0e146022016-06-21 10:32:02 +0300680 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
Felipe Balbiea53b882012-02-17 12:10:04 +0200681 }
682
Felipe Balbiaa3342c2016-03-14 11:01:31 +0200683 while (!list_empty(&dep->pending_list)) {
684 req = next_request(&dep->pending_list);
Felipe Balbi72246da2011-08-19 18:10:58 +0300685
Sebastian Andrzej Siewior624407f2011-08-29 13:56:37 +0200686 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
Felipe Balbi72246da2011-08-19 18:10:58 +0300687 }
Felipe Balbi72246da2011-08-19 18:10:58 +0300688}
689
690/**
691 * __dwc3_gadget_ep_disable - Disables a HW endpoint
692 * @dep: the endpoint to disable
693 *
Sebastian Andrzej Siewior624407f2011-08-29 13:56:37 +0200694 * This function also removes requests which are currently processed ny the
695 * hardware and those which are not yet scheduled.
696 * Caller should take care of locking.
Felipe Balbi72246da2011-08-19 18:10:58 +0300697 */
Felipe Balbi72246da2011-08-19 18:10:58 +0300698static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
699{
700 struct dwc3 *dwc = dep->dwc;
701 u32 reg;
702
Felipe Balbi2870e502016-11-03 13:53:29 +0200703 trace_dwc3_gadget_ep_disable(dep);
Felipe Balbi7eaeac52015-07-20 14:46:15 -0500704
Sebastian Andrzej Siewior624407f2011-08-29 13:56:37 +0200705 dwc3_remove_requests(dwc, dep);
Felipe Balbi72246da2011-08-19 18:10:58 +0300706
Felipe Balbi687ef982014-04-16 10:30:33 -0500707 /* make sure HW endpoint isn't stalled */
708 if (dep->flags & DWC3_EP_STALL)
Felipe Balbi7a608552014-09-24 14:19:52 -0500709 __dwc3_gadget_ep_set_halt(dep, 0, false);
Felipe Balbi687ef982014-04-16 10:30:33 -0500710
Felipe Balbi72246da2011-08-19 18:10:58 +0300711 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
712 reg &= ~DWC3_DALEPENA_EP(dep->number);
713 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
714
Felipe Balbi879631a2011-09-30 10:58:47 +0300715 dep->stream_capable = false;
Ido Shayevitzf9c56cd2012-02-08 13:56:48 +0200716 dep->endpoint.desc = NULL;
Felipe Balbic90bfae2011-11-29 13:11:21 +0200717 dep->comp_desc = NULL;
Felipe Balbi72246da2011-08-19 18:10:58 +0300718 dep->type = 0;
Baolin Wang76a638f2016-10-31 19:38:36 +0800719 dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
Felipe Balbi72246da2011-08-19 18:10:58 +0300720
721 return 0;
722}
723
724/* -------------------------------------------------------------------------- */
725
726static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
727 const struct usb_endpoint_descriptor *desc)
728{
729 return -EINVAL;
730}
731
732static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
733{
734 return -EINVAL;
735}
736
737/* -------------------------------------------------------------------------- */
738
739static int dwc3_gadget_ep_enable(struct usb_ep *ep,
740 const struct usb_endpoint_descriptor *desc)
741{
742 struct dwc3_ep *dep;
743 struct dwc3 *dwc;
744 unsigned long flags;
745 int ret;
746
747 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
748 pr_debug("dwc3: invalid parameters\n");
749 return -EINVAL;
750 }
751
752 if (!desc->wMaxPacketSize) {
753 pr_debug("dwc3: missing wMaxPacketSize\n");
754 return -EINVAL;
755 }
756
757 dep = to_dwc3_ep(ep);
758 dwc = dep->dwc;
759
Felipe Balbi95ca9612015-12-10 13:08:20 -0600760 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
761 "%s is already enabled\n",
762 dep->name))
Felipe Balbic6f83f32012-08-15 12:28:29 +0300763 return 0;
Felipe Balbic6f83f32012-08-15 12:28:29 +0300764
Felipe Balbi72246da2011-08-19 18:10:58 +0300765 spin_lock_irqsave(&dwc->lock, flags);
Paul Zimmerman265b70a2013-12-19 12:38:49 -0600766 ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
Felipe Balbi72246da2011-08-19 18:10:58 +0300767 spin_unlock_irqrestore(&dwc->lock, flags);
768
769 return ret;
770}
771
772static int dwc3_gadget_ep_disable(struct usb_ep *ep)
773{
774 struct dwc3_ep *dep;
775 struct dwc3 *dwc;
776 unsigned long flags;
777 int ret;
778
779 if (!ep) {
780 pr_debug("dwc3: invalid parameters\n");
781 return -EINVAL;
782 }
783
784 dep = to_dwc3_ep(ep);
785 dwc = dep->dwc;
786
Felipe Balbi95ca9612015-12-10 13:08:20 -0600787 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
788 "%s is already disabled\n",
789 dep->name))
Felipe Balbi72246da2011-08-19 18:10:58 +0300790 return 0;
Felipe Balbi72246da2011-08-19 18:10:58 +0300791
Felipe Balbi72246da2011-08-19 18:10:58 +0300792 spin_lock_irqsave(&dwc->lock, flags);
793 ret = __dwc3_gadget_ep_disable(dep);
794 spin_unlock_irqrestore(&dwc->lock, flags);
795
796 return ret;
797}
798
799static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
800 gfp_t gfp_flags)
801{
802 struct dwc3_request *req;
803 struct dwc3_ep *dep = to_dwc3_ep(ep);
Felipe Balbi72246da2011-08-19 18:10:58 +0300804
805 req = kzalloc(sizeof(*req), gfp_flags);
Jingoo Han734d5a52014-07-17 12:45:11 +0900806 if (!req)
Felipe Balbi72246da2011-08-19 18:10:58 +0300807 return NULL;
Felipe Balbi72246da2011-08-19 18:10:58 +0300808
809 req->epnum = dep->number;
810 req->dep = dep;
Felipe Balbi72246da2011-08-19 18:10:58 +0300811
Felipe Balbi68d34c82016-05-30 13:34:58 +0300812 dep->allocated_requests++;
813
Felipe Balbi2c4cbe6e52014-04-30 17:45:10 -0500814 trace_dwc3_alloc_request(req);
815
Felipe Balbi72246da2011-08-19 18:10:58 +0300816 return &req->request;
817}
818
819static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
820 struct usb_request *request)
821{
822 struct dwc3_request *req = to_dwc3_request(request);
Felipe Balbi68d34c82016-05-30 13:34:58 +0300823 struct dwc3_ep *dep = to_dwc3_ep(ep);
Felipe Balbi72246da2011-08-19 18:10:58 +0300824
Felipe Balbi68d34c82016-05-30 13:34:58 +0300825 dep->allocated_requests--;
Felipe Balbi2c4cbe6e52014-04-30 17:45:10 -0500826 trace_dwc3_free_request(req);
Felipe Balbi72246da2011-08-19 18:10:58 +0300827 kfree(req);
828}
829
Felipe Balbi2c78c022016-08-12 13:13:10 +0300830static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);
831
Felipe Balbic71fc372011-11-22 11:37:34 +0200832/**
833 * dwc3_prepare_one_trb - setup one TRB from one request
834 * @dep: endpoint for which this request is prepared
835 * @req: dwc3_request pointer
836 */
Felipe Balbi68e823e2011-11-28 12:25:01 +0200837static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
Felipe Balbieeb720f2011-11-28 12:46:59 +0200838 struct dwc3_request *req, dma_addr_t dma,
Felipe Balbi4bc48c92016-08-10 16:04:33 +0300839 unsigned length, unsigned chain, unsigned node)
Felipe Balbic71fc372011-11-22 11:37:34 +0200840{
Felipe Balbif6bafc62012-02-06 11:04:53 +0200841 struct dwc3_trb *trb;
Felipe Balbi6b9018d2016-09-22 11:01:01 +0300842 struct dwc3 *dwc = dep->dwc;
843 struct usb_gadget *gadget = &dwc->gadget;
844 enum usb_device_speed speed = gadget->speed;
Felipe Balbic71fc372011-11-22 11:37:34 +0200845
Felipe Balbi4faf7552016-04-05 13:14:31 +0300846 trb = &dep->trb_pool[dep->trb_enqueue];
Felipe Balbic71fc372011-11-22 11:37:34 +0200847
Felipe Balbieeb720f2011-11-28 12:46:59 +0200848 if (!req->trb) {
Felipe Balbiaa3342c2016-03-14 11:01:31 +0200849 dwc3_gadget_move_started_request(req);
Felipe Balbif6bafc62012-02-06 11:04:53 +0200850 req->trb = trb;
851 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
Felipe Balbia9c3ca52016-10-05 14:24:37 +0300852 dep->queued_requests++;
Felipe Balbieeb720f2011-11-28 12:46:59 +0200853 }
Felipe Balbic71fc372011-11-22 11:37:34 +0200854
Felipe Balbief966b92016-04-05 13:09:51 +0300855 dwc3_ep_inc_enq(dep);
Pratyush Anande5ba5ec2013-01-14 15:59:37 +0530856
Felipe Balbif6bafc62012-02-06 11:04:53 +0200857 trb->size = DWC3_TRB_SIZE_LENGTH(length);
858 trb->bpl = lower_32_bits(dma);
859 trb->bph = upper_32_bits(dma);
Felipe Balbic71fc372011-11-22 11:37:34 +0200860
Ido Shayevitz16e78db2012-03-12 20:25:24 +0200861 switch (usb_endpoint_type(dep->endpoint.desc)) {
Felipe Balbic71fc372011-11-22 11:37:34 +0200862 case USB_ENDPOINT_XFER_CONTROL:
Felipe Balbif6bafc62012-02-06 11:04:53 +0200863 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
Felipe Balbic71fc372011-11-22 11:37:34 +0200864 break;
865
866 case USB_ENDPOINT_XFER_ISOC:
Felipe Balbi6b9018d2016-09-22 11:01:01 +0300867 if (!node) {
Pratyush Anande5ba5ec2013-01-14 15:59:37 +0530868 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
Felipe Balbi6b9018d2016-09-22 11:01:01 +0300869
870 if (speed == USB_SPEED_HIGH) {
871 struct usb_ep *ep = &dep->endpoint;
872 trb->size |= DWC3_TRB_SIZE_PCM1(ep->mult - 1);
873 }
874 } else {
Pratyush Anande5ba5ec2013-01-14 15:59:37 +0530875 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
Felipe Balbi6b9018d2016-09-22 11:01:01 +0300876 }
Felipe Balbica4d44e2016-03-10 13:53:27 +0200877
878 /* always enable Interrupt on Missed ISOC */
879 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
Felipe Balbic71fc372011-11-22 11:37:34 +0200880 break;
881
882 case USB_ENDPOINT_XFER_BULK:
883 case USB_ENDPOINT_XFER_INT:
Felipe Balbif6bafc62012-02-06 11:04:53 +0200884 trb->ctrl = DWC3_TRBCTL_NORMAL;
Felipe Balbic71fc372011-11-22 11:37:34 +0200885 break;
886 default:
887 /*
888 * This is only possible with faulty memory because we
889 * checked it already :)
890 */
Felipe Balbi0a695d42016-10-07 11:20:01 +0300891 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
892 usb_endpoint_type(dep->endpoint.desc));
Felipe Balbic71fc372011-11-22 11:37:34 +0200893 }
894
Felipe Balbica4d44e2016-03-10 13:53:27 +0200895 /* always enable Continue on Short Packet */
Felipe Balbic9508c82016-10-05 14:26:23 +0300896 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
Felipe Balbi58f29032016-10-06 17:10:39 +0300897 trb->ctrl |= DWC3_TRB_CTRL_CSP;
Felipe Balbif3af3652013-12-13 14:19:33 -0600898
Felipe Balbic9508c82016-10-05 14:26:23 +0300899 if (req->request.short_not_ok)
900 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
901 }
902
Felipe Balbi2c78c022016-08-12 13:13:10 +0300903 if ((!req->request.no_interrupt && !chain) ||
904 (dwc3_calc_trbs_left(dep) == 0))
Felipe Balbic9508c82016-10-05 14:26:23 +0300905 trb->ctrl |= DWC3_TRB_CTRL_IOC;
Felipe Balbica4d44e2016-03-10 13:53:27 +0200906
Pratyush Anande5ba5ec2013-01-14 15:59:37 +0530907 if (chain)
908 trb->ctrl |= DWC3_TRB_CTRL_CHN;
909
Ido Shayevitz16e78db2012-03-12 20:25:24 +0200910 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
Felipe Balbif6bafc62012-02-06 11:04:53 +0200911 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
912
913 trb->ctrl |= DWC3_TRB_CTRL_HWO;
Felipe Balbi2c4cbe6e52014-04-30 17:45:10 -0500914
915 trace_dwc3_prepare_trb(dep, trb);
Felipe Balbic71fc372011-11-22 11:37:34 +0200916}
917
John Youn361572b2016-05-19 17:26:17 -0700918/**
919 * dwc3_ep_prev_trb() - Returns the previous TRB in the ring
920 * @dep: The endpoint with the TRB ring
921 * @index: The index of the current TRB in the ring
922 *
923 * Returns the TRB prior to the one pointed to by the index. If the
924 * index is 0, we will wrap backwards, skip the link TRB, and return
925 * the one just before that.
926 */
927static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
928{
Felipe Balbi45438a02016-08-11 12:26:59 +0300929 u8 tmp = index;
John Youn361572b2016-05-19 17:26:17 -0700930
Felipe Balbi45438a02016-08-11 12:26:59 +0300931 if (!tmp)
932 tmp = DWC3_TRB_NUM - 1;
933
934 return &dep->trb_pool[tmp - 1];
John Youn361572b2016-05-19 17:26:17 -0700935}
936
Felipe Balbic4233572016-05-12 14:08:34 +0300937static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
938{
939 struct dwc3_trb *tmp;
John Youn32db3d92016-05-19 17:26:12 -0700940 u8 trbs_left;
Felipe Balbic4233572016-05-12 14:08:34 +0300941
942 /*
943 * If enqueue & dequeue are equal than it is either full or empty.
944 *
945 * One way to know for sure is if the TRB right before us has HWO bit
946 * set or not. If it has, then we're definitely full and can't fit any
947 * more transfers in our ring.
948 */
949 if (dep->trb_enqueue == dep->trb_dequeue) {
John Youn361572b2016-05-19 17:26:17 -0700950 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
951 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
952 return 0;
Felipe Balbic4233572016-05-12 14:08:34 +0300953
954 return DWC3_TRB_NUM - 1;
955 }
956
John Youn9d7aba72016-08-26 18:43:01 -0700957 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
John Youn3de2685f2016-05-23 11:32:45 -0700958 trbs_left &= (DWC3_TRB_NUM - 1);
John Youn32db3d92016-05-19 17:26:12 -0700959
John Youn9d7aba72016-08-26 18:43:01 -0700960 if (dep->trb_dequeue < dep->trb_enqueue)
961 trbs_left--;
962
John Youn32db3d92016-05-19 17:26:12 -0700963 return trbs_left;
Felipe Balbic4233572016-05-12 14:08:34 +0300964}
965
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300966static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
Felipe Balbi7ae7df42016-08-24 14:37:22 +0300967 struct dwc3_request *req)
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300968{
Felipe Balbi1f512112016-08-12 13:17:27 +0300969 struct scatterlist *sg = req->sg;
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300970 struct scatterlist *s;
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300971 unsigned int length;
972 dma_addr_t dma;
973 int i;
974
Felipe Balbi1f512112016-08-12 13:17:27 +0300975 for_each_sg(sg, s, req->num_pending_sgs, i) {
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300976 unsigned chain = true;
977
978 length = sg_dma_len(s);
979 dma = sg_dma_address(s);
980
Felipe Balbi4bc48c92016-08-10 16:04:33 +0300981 if (sg_is_last(s))
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300982 chain = false;
983
984 dwc3_prepare_one_trb(dep, req, dma, length,
Felipe Balbi4bc48c92016-08-10 16:04:33 +0300985 chain, i);
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300986
Felipe Balbi7ae7df42016-08-24 14:37:22 +0300987 if (!dwc3_calc_trbs_left(dep))
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300988 break;
989 }
990}
991
992static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
Felipe Balbi7ae7df42016-08-24 14:37:22 +0300993 struct dwc3_request *req)
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300994{
Felipe Balbi5ee85d82016-05-13 12:42:44 +0300995 unsigned int length;
996 dma_addr_t dma;
997
998 dma = req->request.dma;
999 length = req->request.length;
1000
Felipe Balbi5ee85d82016-05-13 12:42:44 +03001001 dwc3_prepare_one_trb(dep, req, dma, length,
Felipe Balbi4bc48c92016-08-10 16:04:33 +03001002 false, 0);
Felipe Balbi5ee85d82016-05-13 12:42:44 +03001003}
1004
Felipe Balbi72246da2011-08-19 18:10:58 +03001005/*
1006 * dwc3_prepare_trbs - setup TRBs from requests
1007 * @dep: endpoint for which requests are being prepared
Felipe Balbi72246da2011-08-19 18:10:58 +03001008 *
Paul Zimmerman1d046792012-02-15 18:56:56 -08001009 * The function goes through the requests list and sets up TRBs for the
1010 * transfers. The function returns once there are no more TRBs available or
1011 * it runs out of requests.
Felipe Balbi72246da2011-08-19 18:10:58 +03001012 */
Felipe Balbic4233572016-05-12 14:08:34 +03001013static void dwc3_prepare_trbs(struct dwc3_ep *dep)
Felipe Balbi72246da2011-08-19 18:10:58 +03001014{
Felipe Balbi68e823e2011-11-28 12:25:01 +02001015 struct dwc3_request *req, *n;
Felipe Balbi72246da2011-08-19 18:10:58 +03001016
1017 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1018
Felipe Balbi7ae7df42016-08-24 14:37:22 +03001019 if (!dwc3_calc_trbs_left(dep))
John Youn89bc8562016-05-19 17:26:10 -07001020 return;
Felipe Balbi72246da2011-08-19 18:10:58 +03001021
Felipe Balbid86c5a62016-10-25 13:48:52 +03001022 /*
1023 * We can get in a situation where there's a request in the started list
1024 * but there weren't enough TRBs to fully kick it in the first time
1025 * around, so it has been waiting for more TRBs to be freed up.
1026 *
1027 * In that case, we should check if we have a request with pending_sgs
1028 * in the started list and prepare TRBs for that request first,
1029 * otherwise we will prepare TRBs completely out of order and that will
1030 * break things.
1031 */
1032 list_for_each_entry(req, &dep->started_list, list) {
1033 if (req->num_pending_sgs > 0)
1034 dwc3_prepare_one_trb_sg(dep, req);
1035
1036 if (!dwc3_calc_trbs_left(dep))
1037 return;
1038 }
1039
Felipe Balbiaa3342c2016-03-14 11:01:31 +02001040 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
Felipe Balbi1f512112016-08-12 13:17:27 +03001041 if (req->num_pending_sgs > 0)
Felipe Balbi7ae7df42016-08-24 14:37:22 +03001042 dwc3_prepare_one_trb_sg(dep, req);
Felipe Balbi5ee85d82016-05-13 12:42:44 +03001043 else
Felipe Balbi7ae7df42016-08-24 14:37:22 +03001044 dwc3_prepare_one_trb_linear(dep, req);
Felipe Balbi72246da2011-08-19 18:10:58 +03001045
Felipe Balbi7ae7df42016-08-24 14:37:22 +03001046 if (!dwc3_calc_trbs_left(dep))
Felipe Balbi5ee85d82016-05-13 12:42:44 +03001047 return;
Felipe Balbi72246da2011-08-19 18:10:58 +03001048 }
Felipe Balbi72246da2011-08-19 18:10:58 +03001049}
1050
Felipe Balbi4fae2e32016-05-12 16:53:59 +03001051static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param)
Felipe Balbi72246da2011-08-19 18:10:58 +03001052{
1053 struct dwc3_gadget_ep_cmd_params params;
1054 struct dwc3_request *req;
Felipe Balbi4fae2e32016-05-12 16:53:59 +03001055 int starting;
Felipe Balbi72246da2011-08-19 18:10:58 +03001056 int ret;
1057 u32 cmd;
1058
Felipe Balbi4fae2e32016-05-12 16:53:59 +03001059 starting = !(dep->flags & DWC3_EP_BUSY);
Felipe Balbi72246da2011-08-19 18:10:58 +03001060
Felipe Balbi4fae2e32016-05-12 16:53:59 +03001061 dwc3_prepare_trbs(dep);
1062 req = next_request(&dep->started_list);
Felipe Balbi72246da2011-08-19 18:10:58 +03001063 if (!req) {
1064 dep->flags |= DWC3_EP_PENDING_REQUEST;
1065 return 0;
1066 }
1067
1068 memset(&params, 0, sizeof(params));
Felipe Balbi72246da2011-08-19 18:10:58 +03001069
Felipe Balbi4fae2e32016-05-12 16:53:59 +03001070 if (starting) {
Pratyush Anand1877d6c2013-01-14 15:59:36 +05301071 params.param0 = upper_32_bits(req->trb_dma);
1072 params.param1 = lower_32_bits(req->trb_dma);
Felipe Balbib6b1c6d2016-05-30 13:29:35 +03001073 cmd = DWC3_DEPCMD_STARTTRANSFER |
1074 DWC3_DEPCMD_PARAM(cmd_param);
Pratyush Anand1877d6c2013-01-14 15:59:36 +05301075 } else {
Felipe Balbib6b1c6d2016-05-30 13:29:35 +03001076 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1077 DWC3_DEPCMD_PARAM(dep->resource_index);
Pratyush Anand1877d6c2013-01-14 15:59:36 +05301078 }
Felipe Balbi72246da2011-08-19 18:10:58 +03001079
Felipe Balbi2cd47182016-04-12 16:42:43 +03001080 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
Felipe Balbi72246da2011-08-19 18:10:58 +03001081 if (ret < 0) {
Felipe Balbi72246da2011-08-19 18:10:58 +03001082 /*
1083 * FIXME we need to iterate over the list of requests
1084 * here and stop, unmap, free and del each of the linked
Paul Zimmerman1d046792012-02-15 18:56:56 -08001085 * requests instead of what we do now.
Felipe Balbi72246da2011-08-19 18:10:58 +03001086 */
Felipe Balbi15b8d932016-09-22 10:59:12 +03001087 dwc3_gadget_giveback(dep, req, ret);
Felipe Balbi72246da2011-08-19 18:10:58 +03001088 return ret;
1089 }
1090
1091 dep->flags |= DWC3_EP_BUSY;
Felipe Balbi25b8ff62011-11-04 12:32:47 +02001092
Felipe Balbi4fae2e32016-05-12 16:53:59 +03001093 if (starting) {
Felipe Balbi2eb88012016-04-12 16:53:39 +03001094 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
Felipe Balbib4996a82012-06-06 12:04:13 +03001095 WARN_ON_ONCE(!dep->resource_index);
Paul Zimmermanf898ae02012-03-29 18:16:54 +00001096 }
Felipe Balbi25b8ff62011-11-04 12:32:47 +02001097
Felipe Balbi72246da2011-08-19 18:10:58 +03001098 return 0;
1099}
1100
Felipe Balbi6cb2e4e2016-10-21 13:07:09 +03001101static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1102{
1103 u32 reg;
1104
1105 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1106 return DWC3_DSTS_SOFFN(reg);
1107}
1108
Pratyush Anandd6d6ec72012-05-25 18:54:56 +05301109static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1110 struct dwc3_ep *dep, u32 cur_uf)
1111{
1112 u32 uf;
1113
Felipe Balbiaa3342c2016-03-14 11:01:31 +02001114 if (list_empty(&dep->pending_list)) {
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02001115 dev_info(dwc->dev, "%s: ran out of requests\n",
Felipe Balbi73815282015-01-27 13:48:14 -06001116 dep->name);
Pratyush Anandf4a53c52012-08-30 12:21:43 +05301117 dep->flags |= DWC3_EP_PENDING_REQUEST;
Pratyush Anandd6d6ec72012-05-25 18:54:56 +05301118 return;
1119 }
1120
1121 /* 4 micro frames in the future */
1122 uf = cur_uf + dep->interval * 4;
1123
Felipe Balbi4fae2e32016-05-12 16:53:59 +03001124 __dwc3_gadget_kick_transfer(dep, uf);
Pratyush Anandd6d6ec72012-05-25 18:54:56 +05301125}
1126
1127static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1128 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1129{
1130 u32 cur_uf, mask;
1131
1132 mask = ~(dep->interval - 1);
1133 cur_uf = event->parameters & mask;
1134
1135 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1136}
1137
Felipe Balbi72246da2011-08-19 18:10:58 +03001138static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1139{
Felipe Balbi0fc9a1b2011-12-19 11:32:34 +02001140 struct dwc3 *dwc = dep->dwc;
1141 int ret;
1142
Felipe Balbibb423982015-11-16 15:31:21 -06001143 if (!dep->endpoint.desc) {
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02001144 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1145 dep->name);
Felipe Balbibb423982015-11-16 15:31:21 -06001146 return -ESHUTDOWN;
1147 }
1148
1149 if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
1150 &req->request, req->dep->name)) {
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02001151 dev_err(dwc->dev, "%s: request %p belongs to '%s'\n",
1152 dep->name, &req->request, req->dep->name);
Felipe Balbibb423982015-11-16 15:31:21 -06001153 return -EINVAL;
1154 }
1155
Felipe Balbifc8bb912016-05-16 13:14:48 +03001156 pm_runtime_get(dwc->dev);
1157
Felipe Balbi72246da2011-08-19 18:10:58 +03001158 req->request.actual = 0;
1159 req->request.status = -EINPROGRESS;
1160 req->direction = dep->direction;
1161 req->epnum = dep->number;
1162
Felipe Balbife84f522015-09-01 09:01:38 -05001163 trace_dwc3_ep_queue(req);
1164
Felipe Balbi0fc9a1b2011-12-19 11:32:34 +02001165 ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1166 dep->direction);
1167 if (ret)
1168 return ret;
1169
Felipe Balbi1f512112016-08-12 13:17:27 +03001170 req->sg = req->request.sg;
1171 req->num_pending_sgs = req->request.num_mapped_sgs;
1172
Felipe Balbiaa3342c2016-03-14 11:01:31 +02001173 list_add_tail(&req->list, &dep->pending_list);
Felipe Balbi72246da2011-08-19 18:10:58 +03001174
Felipe Balbid889c232016-09-29 15:44:29 +03001175 /*
1176 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1177 * wait for a XferNotReady event so we will know what's the current
1178 * (micro-)frame number.
1179 *
1180 * Without this trick, we are very, very likely gonna get Bus Expiry
1181 * errors which will force us issue EndTransfer command.
1182 */
1183 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
Felipe Balbi6cb2e4e2016-10-21 13:07:09 +03001184 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1185 if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1186 dwc3_stop_active_transfer(dwc, dep->number, true);
1187 dep->flags = DWC3_EP_ENABLED;
1188 } else {
1189 u32 cur_uf;
1190
1191 cur_uf = __dwc3_gadget_get_frame(dwc);
1192 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1193 }
Felipe Balbi08a36b52016-08-11 14:27:52 +03001194 }
1195 return 0;
Felipe Balbib511e5e2012-06-06 12:00:50 +03001196 }
1197
Felipe Balbi594e1212016-08-24 14:38:10 +03001198 if (!dwc3_calc_trbs_left(dep))
1199 return 0;
Felipe Balbib997ada2012-07-26 13:26:50 +03001200
Felipe Balbi08a36b52016-08-11 14:27:52 +03001201 ret = __dwc3_gadget_kick_transfer(dep, 0);
Felipe Balbia8f32812015-09-16 10:40:07 -05001202 if (ret == -EBUSY)
1203 ret = 0;
1204
1205 return ret;
Felipe Balbi72246da2011-08-19 18:10:58 +03001206}
1207
Felipe Balbi04c03d12015-12-02 10:06:45 -06001208static void __dwc3_gadget_ep_zlp_complete(struct usb_ep *ep,
1209 struct usb_request *request)
1210{
1211 dwc3_gadget_ep_free_request(ep, request);
1212}
1213
1214static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep)
1215{
1216 struct dwc3_request *req;
1217 struct usb_request *request;
1218 struct usb_ep *ep = &dep->endpoint;
1219
Felipe Balbi04c03d12015-12-02 10:06:45 -06001220 request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1221 if (!request)
1222 return -ENOMEM;
1223
1224 request->length = 0;
1225 request->buf = dwc->zlp_buf;
1226 request->complete = __dwc3_gadget_ep_zlp_complete;
1227
1228 req = to_dwc3_request(request);
1229
1230 return __dwc3_gadget_ep_queue(dep, req);
1231}
1232
Felipe Balbi72246da2011-08-19 18:10:58 +03001233static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1234 gfp_t gfp_flags)
1235{
1236 struct dwc3_request *req = to_dwc3_request(request);
1237 struct dwc3_ep *dep = to_dwc3_ep(ep);
1238 struct dwc3 *dwc = dep->dwc;
1239
1240 unsigned long flags;
1241
1242 int ret;
1243
Zhuang Jin Canfdee4eb2014-09-03 14:26:34 +08001244 spin_lock_irqsave(&dwc->lock, flags);
Felipe Balbi72246da2011-08-19 18:10:58 +03001245 ret = __dwc3_gadget_ep_queue(dep, req);
Felipe Balbi04c03d12015-12-02 10:06:45 -06001246
1247 /*
1248 * Okay, here's the thing, if gadget driver has requested for a ZLP by
1249 * setting request->zero, instead of doing magic, we will just queue an
1250 * extra usb_request ourselves so that it gets handled the same way as
1251 * any other request.
1252 */
John Yound92618982015-12-22 12:23:20 -08001253 if (ret == 0 && request->zero && request->length &&
1254 (request->length % ep->maxpacket == 0))
Felipe Balbi04c03d12015-12-02 10:06:45 -06001255 ret = __dwc3_gadget_ep_queue_zlp(dwc, dep);
1256
Felipe Balbi72246da2011-08-19 18:10:58 +03001257 spin_unlock_irqrestore(&dwc->lock, flags);
1258
1259 return ret;
1260}
1261
1262static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1263 struct usb_request *request)
1264{
1265 struct dwc3_request *req = to_dwc3_request(request);
1266 struct dwc3_request *r = NULL;
1267
1268 struct dwc3_ep *dep = to_dwc3_ep(ep);
1269 struct dwc3 *dwc = dep->dwc;
1270
1271 unsigned long flags;
1272 int ret = 0;
1273
Felipe Balbi2c4cbe6e52014-04-30 17:45:10 -05001274 trace_dwc3_ep_dequeue(req);
1275
Felipe Balbi72246da2011-08-19 18:10:58 +03001276 spin_lock_irqsave(&dwc->lock, flags);
1277
Felipe Balbiaa3342c2016-03-14 11:01:31 +02001278 list_for_each_entry(r, &dep->pending_list, list) {
Felipe Balbi72246da2011-08-19 18:10:58 +03001279 if (r == req)
1280 break;
1281 }
1282
1283 if (r != req) {
Felipe Balbiaa3342c2016-03-14 11:01:31 +02001284 list_for_each_entry(r, &dep->started_list, list) {
Felipe Balbi72246da2011-08-19 18:10:58 +03001285 if (r == req)
1286 break;
1287 }
1288 if (r == req) {
1289 /* wait until it is processed */
Paul Zimmermanb992e682012-04-27 14:17:35 +03001290 dwc3_stop_active_transfer(dwc, dep->number, true);
Pratyush Anande8d4e8b2012-06-15 11:54:00 +05301291 goto out1;
Felipe Balbi72246da2011-08-19 18:10:58 +03001292 }
1293 dev_err(dwc->dev, "request %p was not queued to %s\n",
1294 request, ep->name);
1295 ret = -EINVAL;
1296 goto out0;
1297 }
1298
Pratyush Anande8d4e8b2012-06-15 11:54:00 +05301299out1:
Felipe Balbi72246da2011-08-19 18:10:58 +03001300 /* giveback the request */
1301 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1302
1303out0:
1304 spin_unlock_irqrestore(&dwc->lock, flags);
1305
1306 return ret;
1307}
1308
Felipe Balbi7a608552014-09-24 14:19:52 -05001309int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
Felipe Balbi72246da2011-08-19 18:10:58 +03001310{
1311 struct dwc3_gadget_ep_cmd_params params;
1312 struct dwc3 *dwc = dep->dwc;
1313 int ret;
1314
Felipe Balbi5ad02fb2014-09-24 10:48:26 -05001315 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1316 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1317 return -EINVAL;
1318 }
1319
Felipe Balbi72246da2011-08-19 18:10:58 +03001320 memset(&params, 0x00, sizeof(params));
1321
1322 if (value) {
Felipe Balbi69450c42016-05-30 13:37:02 +03001323 struct dwc3_trb *trb;
1324
1325 unsigned transfer_in_flight;
1326 unsigned started;
1327
1328 if (dep->number > 1)
1329 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1330 else
1331 trb = &dwc->ep0_trb[dep->trb_enqueue];
1332
1333 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1334 started = !list_empty(&dep->started_list);
1335
1336 if (!protocol && ((dep->direction && transfer_in_flight) ||
1337 (!dep->direction && started))) {
Felipe Balbi7a608552014-09-24 14:19:52 -05001338 return -EAGAIN;
1339 }
1340
Felipe Balbi2cd47182016-04-12 16:42:43 +03001341 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1342 &params);
Felipe Balbi72246da2011-08-19 18:10:58 +03001343 if (ret)
Dan Carpenter3f892042014-03-07 14:20:22 +03001344 dev_err(dwc->dev, "failed to set STALL on %s\n",
Felipe Balbi72246da2011-08-19 18:10:58 +03001345 dep->name);
1346 else
1347 dep->flags |= DWC3_EP_STALL;
1348 } else {
Felipe Balbi2cd47182016-04-12 16:42:43 +03001349
John Youn50c763f2016-05-31 17:49:56 -07001350 ret = dwc3_send_clear_stall_ep_cmd(dep);
Felipe Balbi72246da2011-08-19 18:10:58 +03001351 if (ret)
Dan Carpenter3f892042014-03-07 14:20:22 +03001352 dev_err(dwc->dev, "failed to clear STALL on %s\n",
Felipe Balbi72246da2011-08-19 18:10:58 +03001353 dep->name);
1354 else
Alan Sterna535d812013-11-01 12:05:12 -04001355 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
Felipe Balbi72246da2011-08-19 18:10:58 +03001356 }
Paul Zimmerman52754552011-09-30 10:58:44 +03001357
Felipe Balbi72246da2011-08-19 18:10:58 +03001358 return ret;
1359}
1360
1361static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1362{
1363 struct dwc3_ep *dep = to_dwc3_ep(ep);
1364 struct dwc3 *dwc = dep->dwc;
1365
1366 unsigned long flags;
1367
1368 int ret;
1369
1370 spin_lock_irqsave(&dwc->lock, flags);
Felipe Balbi7a608552014-09-24 14:19:52 -05001371 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
Felipe Balbi72246da2011-08-19 18:10:58 +03001372 spin_unlock_irqrestore(&dwc->lock, flags);
1373
1374 return ret;
1375}
1376
1377static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1378{
1379 struct dwc3_ep *dep = to_dwc3_ep(ep);
Paul Zimmerman249a4562012-02-24 17:32:16 -08001380 struct dwc3 *dwc = dep->dwc;
1381 unsigned long flags;
Felipe Balbi95aa4e82014-09-24 10:50:14 -05001382 int ret;
Felipe Balbi72246da2011-08-19 18:10:58 +03001383
Paul Zimmerman249a4562012-02-24 17:32:16 -08001384 spin_lock_irqsave(&dwc->lock, flags);
Felipe Balbi72246da2011-08-19 18:10:58 +03001385 dep->flags |= DWC3_EP_WEDGE;
1386
Pratyush Anand08f0d962012-06-25 22:40:43 +05301387 if (dep->number == 0 || dep->number == 1)
Felipe Balbi95aa4e82014-09-24 10:50:14 -05001388 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
Pratyush Anand08f0d962012-06-25 22:40:43 +05301389 else
Felipe Balbi7a608552014-09-24 14:19:52 -05001390 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
Felipe Balbi95aa4e82014-09-24 10:50:14 -05001391 spin_unlock_irqrestore(&dwc->lock, flags);
1392
1393 return ret;
Felipe Balbi72246da2011-08-19 18:10:58 +03001394}
1395
1396/* -------------------------------------------------------------------------- */
1397
1398static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1399 .bLength = USB_DT_ENDPOINT_SIZE,
1400 .bDescriptorType = USB_DT_ENDPOINT,
1401 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1402};
1403
1404static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1405 .enable = dwc3_gadget_ep0_enable,
1406 .disable = dwc3_gadget_ep0_disable,
1407 .alloc_request = dwc3_gadget_ep_alloc_request,
1408 .free_request = dwc3_gadget_ep_free_request,
1409 .queue = dwc3_gadget_ep0_queue,
1410 .dequeue = dwc3_gadget_ep_dequeue,
Pratyush Anand08f0d962012-06-25 22:40:43 +05301411 .set_halt = dwc3_gadget_ep0_set_halt,
Felipe Balbi72246da2011-08-19 18:10:58 +03001412 .set_wedge = dwc3_gadget_ep_set_wedge,
1413};
1414
1415static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1416 .enable = dwc3_gadget_ep_enable,
1417 .disable = dwc3_gadget_ep_disable,
1418 .alloc_request = dwc3_gadget_ep_alloc_request,
1419 .free_request = dwc3_gadget_ep_free_request,
1420 .queue = dwc3_gadget_ep_queue,
1421 .dequeue = dwc3_gadget_ep_dequeue,
1422 .set_halt = dwc3_gadget_ep_set_halt,
1423 .set_wedge = dwc3_gadget_ep_set_wedge,
1424};
1425
1426/* -------------------------------------------------------------------------- */
1427
1428static int dwc3_gadget_get_frame(struct usb_gadget *g)
1429{
1430 struct dwc3 *dwc = gadget_to_dwc(g);
Felipe Balbi72246da2011-08-19 18:10:58 +03001431
Felipe Balbi6cb2e4e2016-10-21 13:07:09 +03001432 return __dwc3_gadget_get_frame(dwc);
Felipe Balbi72246da2011-08-19 18:10:58 +03001433}
1434
Felipe Balbi218ef7b2016-04-04 11:24:04 +03001435static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
Felipe Balbi72246da2011-08-19 18:10:58 +03001436{
Nicolas Saenz Julienned6011f62016-08-16 10:22:38 +01001437 int retries;
Felipe Balbi72246da2011-08-19 18:10:58 +03001438
Felipe Balbi218ef7b2016-04-04 11:24:04 +03001439 int ret;
Felipe Balbi72246da2011-08-19 18:10:58 +03001440 u32 reg;
1441
Felipe Balbi72246da2011-08-19 18:10:58 +03001442 u8 link_state;
1443 u8 speed;
1444
Felipe Balbi72246da2011-08-19 18:10:58 +03001445 /*
1446 * According to the Databook Remote wakeup request should
1447 * be issued only when the device is in early suspend state.
1448 *
1449 * We can check that via USB Link State bits in DSTS register.
1450 */
1451 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1452
1453 speed = reg & DWC3_DSTS_CONNECTSPD;
John Younee5cd412016-02-05 17:08:45 -08001454 if ((speed == DWC3_DSTS_SUPERSPEED) ||
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02001455 (speed == DWC3_DSTS_SUPERSPEED_PLUS))
Felipe Balbi6b742892016-05-13 10:19:42 +03001456 return 0;
Felipe Balbi72246da2011-08-19 18:10:58 +03001457
1458 link_state = DWC3_DSTS_USBLNKST(reg);
1459
1460 switch (link_state) {
1461 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
1462 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
1463 break;
1464 default:
Felipe Balbi218ef7b2016-04-04 11:24:04 +03001465 return -EINVAL;
Felipe Balbi72246da2011-08-19 18:10:58 +03001466 }
1467
Felipe Balbi8598bde2012-01-02 18:55:57 +02001468 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1469 if (ret < 0) {
1470 dev_err(dwc->dev, "failed to put link in Recovery\n");
Felipe Balbi218ef7b2016-04-04 11:24:04 +03001471 return ret;
Felipe Balbi8598bde2012-01-02 18:55:57 +02001472 }
Felipe Balbi72246da2011-08-19 18:10:58 +03001473
Paul Zimmerman802fde92012-04-27 13:10:52 +03001474 /* Recent versions do this automatically */
1475 if (dwc->revision < DWC3_REVISION_194A) {
1476 /* write zeroes to Link Change Request */
Felipe Balbifcc023c2012-05-24 10:27:56 +03001477 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
Paul Zimmerman802fde92012-04-27 13:10:52 +03001478 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1479 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1480 }
Felipe Balbi72246da2011-08-19 18:10:58 +03001481
Paul Zimmerman1d046792012-02-15 18:56:56 -08001482 /* poll until Link State changes to ON */
Nicolas Saenz Julienned6011f62016-08-16 10:22:38 +01001483 retries = 20000;
Felipe Balbi72246da2011-08-19 18:10:58 +03001484
Nicolas Saenz Julienned6011f62016-08-16 10:22:38 +01001485 while (retries--) {
Felipe Balbi72246da2011-08-19 18:10:58 +03001486 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1487
1488 /* in HS, means ON */
1489 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1490 break;
1491 }
1492
1493 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1494 dev_err(dwc->dev, "failed to send remote wakeup\n");
Felipe Balbi218ef7b2016-04-04 11:24:04 +03001495 return -EINVAL;
Felipe Balbi72246da2011-08-19 18:10:58 +03001496 }
1497
Felipe Balbi218ef7b2016-04-04 11:24:04 +03001498 return 0;
1499}
1500
1501static int dwc3_gadget_wakeup(struct usb_gadget *g)
1502{
1503 struct dwc3 *dwc = gadget_to_dwc(g);
1504 unsigned long flags;
1505 int ret;
1506
1507 spin_lock_irqsave(&dwc->lock, flags);
1508 ret = __dwc3_gadget_wakeup(dwc);
Felipe Balbi72246da2011-08-19 18:10:58 +03001509 spin_unlock_irqrestore(&dwc->lock, flags);
1510
1511 return ret;
1512}
1513
1514static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1515 int is_selfpowered)
1516{
1517 struct dwc3 *dwc = gadget_to_dwc(g);
Paul Zimmerman249a4562012-02-24 17:32:16 -08001518 unsigned long flags;
Felipe Balbi72246da2011-08-19 18:10:58 +03001519
Paul Zimmerman249a4562012-02-24 17:32:16 -08001520 spin_lock_irqsave(&dwc->lock, flags);
Peter Chenbcdea502015-01-28 16:32:40 +08001521 g->is_selfpowered = !!is_selfpowered;
Paul Zimmerman249a4562012-02-24 17:32:16 -08001522 spin_unlock_irqrestore(&dwc->lock, flags);
Felipe Balbi72246da2011-08-19 18:10:58 +03001523
1524 return 0;
1525}
1526
Felipe Balbi7b2a0362013-12-19 13:43:19 -06001527static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
Felipe Balbi72246da2011-08-19 18:10:58 +03001528{
1529 u32 reg;
Sebastian Andrzej Siewior61d58242011-08-29 16:46:38 +02001530 u32 timeout = 500;
Felipe Balbi72246da2011-08-19 18:10:58 +03001531
Felipe Balbifc8bb912016-05-16 13:14:48 +03001532 if (pm_runtime_suspended(dwc->dev))
1533 return 0;
1534
Felipe Balbi72246da2011-08-19 18:10:58 +03001535 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
Felipe Balbi8db7ed12012-01-18 18:32:29 +02001536 if (is_on) {
Paul Zimmerman802fde92012-04-27 13:10:52 +03001537 if (dwc->revision <= DWC3_REVISION_187A) {
1538 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1539 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1540 }
1541
1542 if (dwc->revision >= DWC3_REVISION_194A)
1543 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1544 reg |= DWC3_DCTL_RUN_STOP;
Felipe Balbi7b2a0362013-12-19 13:43:19 -06001545
1546 if (dwc->has_hibernation)
1547 reg |= DWC3_DCTL_KEEP_CONNECT;
1548
Felipe Balbi9fcb3bd2013-02-08 17:55:58 +02001549 dwc->pullups_connected = true;
Felipe Balbi8db7ed12012-01-18 18:32:29 +02001550 } else {
Felipe Balbi72246da2011-08-19 18:10:58 +03001551 reg &= ~DWC3_DCTL_RUN_STOP;
Felipe Balbi7b2a0362013-12-19 13:43:19 -06001552
1553 if (dwc->has_hibernation && !suspend)
1554 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1555
Felipe Balbi9fcb3bd2013-02-08 17:55:58 +02001556 dwc->pullups_connected = false;
Felipe Balbi8db7ed12012-01-18 18:32:29 +02001557 }
Felipe Balbi72246da2011-08-19 18:10:58 +03001558
1559 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1560
1561 do {
1562 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
Felipe Balbib6d4e162016-06-09 16:47:05 +03001563 reg &= DWC3_DSTS_DEVCTRLHLT;
1564 } while (--timeout && !(!is_on ^ !reg));
Felipe Balbif2df6792016-06-09 16:31:34 +03001565
1566 if (!timeout)
1567 return -ETIMEDOUT;
Felipe Balbi72246da2011-08-19 18:10:58 +03001568
Pratyush Anand6f17f742012-07-02 10:21:55 +05301569 return 0;
Felipe Balbi72246da2011-08-19 18:10:58 +03001570}
1571
1572static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1573{
1574 struct dwc3 *dwc = gadget_to_dwc(g);
1575 unsigned long flags;
Pratyush Anand6f17f742012-07-02 10:21:55 +05301576 int ret;
Felipe Balbi72246da2011-08-19 18:10:58 +03001577
1578 is_on = !!is_on;
1579
Baolin Wangbb014732016-10-14 17:11:33 +08001580 /*
1581 * Per databook, when we want to stop the gadget, if a control transfer
1582 * is still in process, complete it and get the core into setup phase.
1583 */
1584 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1585 reinit_completion(&dwc->ep0_in_setup);
1586
1587 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1588 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1589 if (ret == 0) {
1590 dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1591 return -ETIMEDOUT;
1592 }
1593 }
1594
Felipe Balbi72246da2011-08-19 18:10:58 +03001595 spin_lock_irqsave(&dwc->lock, flags);
Felipe Balbi7b2a0362013-12-19 13:43:19 -06001596 ret = dwc3_gadget_run_stop(dwc, is_on, false);
Felipe Balbi72246da2011-08-19 18:10:58 +03001597 spin_unlock_irqrestore(&dwc->lock, flags);
1598
Pratyush Anand6f17f742012-07-02 10:21:55 +05301599 return ret;
Felipe Balbi72246da2011-08-19 18:10:58 +03001600}
1601
Felipe Balbi8698e2a2013-02-08 15:24:04 +02001602static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1603{
1604 u32 reg;
1605
1606 /* Enable all but Start and End of Frame IRQs */
1607 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1608 DWC3_DEVTEN_EVNTOVERFLOWEN |
1609 DWC3_DEVTEN_CMDCMPLTEN |
1610 DWC3_DEVTEN_ERRTICERREN |
1611 DWC3_DEVTEN_WKUPEVTEN |
Felipe Balbi8698e2a2013-02-08 15:24:04 +02001612 DWC3_DEVTEN_CONNECTDONEEN |
1613 DWC3_DEVTEN_USBRSTEN |
1614 DWC3_DEVTEN_DISCONNEVTEN);
1615
Felipe Balbi799e9dc2016-09-23 11:20:40 +03001616 if (dwc->revision < DWC3_REVISION_250A)
1617 reg |= DWC3_DEVTEN_ULSTCNGEN;
1618
Felipe Balbi8698e2a2013-02-08 15:24:04 +02001619 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1620}
1621
1622static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1623{
1624 /* mask all interrupts */
1625 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1626}
1627
1628static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
Felipe Balbib15a7622011-06-30 16:57:15 +03001629static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
Felipe Balbi8698e2a2013-02-08 15:24:04 +02001630
Felipe Balbi4e994722016-05-13 14:09:59 +03001631/**
1632 * dwc3_gadget_setup_nump - Calculate and initialize NUMP field of DCFG
1633 * dwc: pointer to our context structure
1634 *
1635 * The following looks like complex but it's actually very simple. In order to
1636 * calculate the number of packets we can burst at once on OUT transfers, we're
1637 * gonna use RxFIFO size.
1638 *
1639 * To calculate RxFIFO size we need two numbers:
1640 * MDWIDTH = size, in bits, of the internal memory bus
1641 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1642 *
1643 * Given these two numbers, the formula is simple:
1644 *
1645 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1646 *
1647 * 24 bytes is for 3x SETUP packets
1648 * 16 bytes is a clock domain crossing tolerance
1649 *
1650 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1651 */
1652static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1653{
1654 u32 ram2_depth;
1655 u32 mdwidth;
1656 u32 nump;
1657 u32 reg;
1658
1659 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1660 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1661
1662 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1663 nump = min_t(u32, nump, 16);
1664
1665 /* update NumP */
1666 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1667 reg &= ~DWC3_DCFG_NUMP_MASK;
1668 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1669 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1670}
1671
Felipe Balbid7be2952016-05-04 15:49:37 +03001672static int __dwc3_gadget_start(struct dwc3 *dwc)
Felipe Balbi72246da2011-08-19 18:10:58 +03001673{
Felipe Balbi72246da2011-08-19 18:10:58 +03001674 struct dwc3_ep *dep;
Felipe Balbi72246da2011-08-19 18:10:58 +03001675 int ret = 0;
1676 u32 reg;
1677
Felipe Balbi72246da2011-08-19 18:10:58 +03001678 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1679 reg &= ~(DWC3_DCFG_SPEED_MASK);
Felipe Balbi07e7f472012-03-23 12:20:31 +02001680
1681 /**
1682 * WORKAROUND: DWC3 revision < 2.20a have an issue
1683 * which would cause metastability state on Run/Stop
1684 * bit if we try to force the IP to USB2-only mode.
1685 *
1686 * Because of that, we cannot configure the IP to any
1687 * speed other than the SuperSpeed
1688 *
1689 * Refers to:
1690 *
1691 * STAR#9000525659: Clock Domain Crossing on DCTL in
1692 * USB 2.0 Mode
1693 */
Felipe Balbif7e846f2013-06-30 14:29:51 +03001694 if (dwc->revision < DWC3_REVISION_220A) {
Felipe Balbi07e7f472012-03-23 12:20:31 +02001695 reg |= DWC3_DCFG_SUPERSPEED;
Felipe Balbif7e846f2013-06-30 14:29:51 +03001696 } else {
1697 switch (dwc->maximum_speed) {
1698 case USB_SPEED_LOW:
John Youn2da9ad72016-05-20 16:34:26 -07001699 reg |= DWC3_DCFG_LOWSPEED;
Felipe Balbif7e846f2013-06-30 14:29:51 +03001700 break;
1701 case USB_SPEED_FULL:
John Youn2da9ad72016-05-20 16:34:26 -07001702 reg |= DWC3_DCFG_FULLSPEED1;
Felipe Balbif7e846f2013-06-30 14:29:51 +03001703 break;
1704 case USB_SPEED_HIGH:
John Youn2da9ad72016-05-20 16:34:26 -07001705 reg |= DWC3_DCFG_HIGHSPEED;
Felipe Balbif7e846f2013-06-30 14:29:51 +03001706 break;
John Youn75808622016-02-05 17:09:13 -08001707 case USB_SPEED_SUPER_PLUS:
John Youn2da9ad72016-05-20 16:34:26 -07001708 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
John Youn75808622016-02-05 17:09:13 -08001709 break;
Felipe Balbif7e846f2013-06-30 14:29:51 +03001710 default:
John Youn77966eb2016-02-19 17:31:01 -08001711 dev_err(dwc->dev, "invalid dwc->maximum_speed (%d)\n",
1712 dwc->maximum_speed);
1713 /* fall through */
1714 case USB_SPEED_SUPER:
1715 reg |= DWC3_DCFG_SUPERSPEED;
1716 break;
Felipe Balbif7e846f2013-06-30 14:29:51 +03001717 }
1718 }
Felipe Balbi72246da2011-08-19 18:10:58 +03001719 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1720
Felipe Balbi2a58f9c2016-04-28 10:56:28 +03001721 /*
1722 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1723 * field instead of letting dwc3 itself calculate that automatically.
1724 *
1725 * This way, we maximize the chances that we'll be able to get several
1726 * bursts of data without going through any sort of endpoint throttling.
1727 */
1728 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1729 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1730 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1731
Felipe Balbi4e994722016-05-13 14:09:59 +03001732 dwc3_gadget_setup_nump(dwc);
1733
Felipe Balbi72246da2011-08-19 18:10:58 +03001734 /* Start with SuperSpeed Default */
1735 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1736
1737 dep = dwc->eps[0];
Paul Zimmerman265b70a2013-12-19 12:38:49 -06001738 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1739 false);
Felipe Balbi72246da2011-08-19 18:10:58 +03001740 if (ret) {
1741 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
Felipe Balbid7be2952016-05-04 15:49:37 +03001742 goto err0;
Felipe Balbi72246da2011-08-19 18:10:58 +03001743 }
1744
1745 dep = dwc->eps[1];
Paul Zimmerman265b70a2013-12-19 12:38:49 -06001746 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1747 false);
Felipe Balbi72246da2011-08-19 18:10:58 +03001748 if (ret) {
1749 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
Felipe Balbid7be2952016-05-04 15:49:37 +03001750 goto err1;
Felipe Balbi72246da2011-08-19 18:10:58 +03001751 }
1752
1753 /* begin to receive SETUP packets */
Felipe Balbic7fcdeb2011-08-27 22:28:36 +03001754 dwc->ep0state = EP0_SETUP_PHASE;
Felipe Balbi72246da2011-08-19 18:10:58 +03001755 dwc3_ep0_out_start(dwc);
1756
Felipe Balbi8698e2a2013-02-08 15:24:04 +02001757 dwc3_gadget_enable_irq(dwc);
1758
Felipe Balbid7be2952016-05-04 15:49:37 +03001759 return 0;
1760
1761err1:
1762 __dwc3_gadget_ep_disable(dwc->eps[0]);
1763
1764err0:
1765 return ret;
1766}
1767
1768static int dwc3_gadget_start(struct usb_gadget *g,
1769 struct usb_gadget_driver *driver)
1770{
1771 struct dwc3 *dwc = gadget_to_dwc(g);
1772 unsigned long flags;
1773 int ret = 0;
1774 int irq;
1775
Roger Quadros9522def2016-06-10 14:48:38 +03001776 irq = dwc->irq_gadget;
Felipe Balbid7be2952016-05-04 15:49:37 +03001777 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1778 IRQF_SHARED, "dwc3", dwc->ev_buf);
1779 if (ret) {
1780 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1781 irq, ret);
1782 goto err0;
1783 }
1784
1785 spin_lock_irqsave(&dwc->lock, flags);
1786 if (dwc->gadget_driver) {
1787 dev_err(dwc->dev, "%s is already bound to %s\n",
1788 dwc->gadget.name,
1789 dwc->gadget_driver->driver.name);
1790 ret = -EBUSY;
1791 goto err1;
1792 }
1793
1794 dwc->gadget_driver = driver;
1795
Felipe Balbifc8bb912016-05-16 13:14:48 +03001796 if (pm_runtime_active(dwc->dev))
1797 __dwc3_gadget_start(dwc);
1798
Felipe Balbi72246da2011-08-19 18:10:58 +03001799 spin_unlock_irqrestore(&dwc->lock, flags);
1800
1801 return 0;
1802
Felipe Balbib0d7ffd2013-06-27 10:00:18 +03001803err1:
Felipe Balbi72246da2011-08-19 18:10:58 +03001804 spin_unlock_irqrestore(&dwc->lock, flags);
Felipe Balbid7be2952016-05-04 15:49:37 +03001805 free_irq(irq, dwc);
Felipe Balbib0d7ffd2013-06-27 10:00:18 +03001806
1807err0:
Felipe Balbi72246da2011-08-19 18:10:58 +03001808 return ret;
1809}
1810
Felipe Balbid7be2952016-05-04 15:49:37 +03001811static void __dwc3_gadget_stop(struct dwc3 *dwc)
1812{
1813 dwc3_gadget_disable_irq(dwc);
1814 __dwc3_gadget_ep_disable(dwc->eps[0]);
1815 __dwc3_gadget_ep_disable(dwc->eps[1]);
1816}
1817
Felipe Balbi22835b82014-10-17 12:05:12 -05001818static int dwc3_gadget_stop(struct usb_gadget *g)
Felipe Balbi72246da2011-08-19 18:10:58 +03001819{
1820 struct dwc3 *dwc = gadget_to_dwc(g);
1821 unsigned long flags;
Baolin Wang76a638f2016-10-31 19:38:36 +08001822 int epnum;
Felipe Balbi72246da2011-08-19 18:10:58 +03001823
1824 spin_lock_irqsave(&dwc->lock, flags);
Baolin Wang76a638f2016-10-31 19:38:36 +08001825
1826 if (pm_runtime_suspended(dwc->dev))
1827 goto out;
1828
Felipe Balbid7be2952016-05-04 15:49:37 +03001829 __dwc3_gadget_stop(dwc);
Baolin Wang76a638f2016-10-31 19:38:36 +08001830
1831 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1832 struct dwc3_ep *dep = dwc->eps[epnum];
1833
1834 if (!dep)
1835 continue;
1836
1837 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1838 continue;
1839
1840 wait_event_lock_irq(dep->wait_end_transfer,
1841 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1842 dwc->lock);
1843 }
1844
1845out:
Felipe Balbi72246da2011-08-19 18:10:58 +03001846 dwc->gadget_driver = NULL;
Felipe Balbi72246da2011-08-19 18:10:58 +03001847 spin_unlock_irqrestore(&dwc->lock, flags);
1848
Felipe Balbi3f308d12016-05-16 14:17:06 +03001849 free_irq(dwc->irq_gadget, dwc->ev_buf);
Felipe Balbib0d7ffd2013-06-27 10:00:18 +03001850
Felipe Balbi72246da2011-08-19 18:10:58 +03001851 return 0;
1852}
Paul Zimmerman802fde92012-04-27 13:10:52 +03001853
Felipe Balbi72246da2011-08-19 18:10:58 +03001854static const struct usb_gadget_ops dwc3_gadget_ops = {
1855 .get_frame = dwc3_gadget_get_frame,
1856 .wakeup = dwc3_gadget_wakeup,
1857 .set_selfpowered = dwc3_gadget_set_selfpowered,
1858 .pullup = dwc3_gadget_pullup,
1859 .udc_start = dwc3_gadget_start,
1860 .udc_stop = dwc3_gadget_stop,
1861};
1862
1863/* -------------------------------------------------------------------------- */
1864
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001865static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1866 u8 num, u32 direction)
Felipe Balbi72246da2011-08-19 18:10:58 +03001867{
1868 struct dwc3_ep *dep;
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001869 u8 i;
Felipe Balbi72246da2011-08-19 18:10:58 +03001870
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001871 for (i = 0; i < num; i++) {
John Yound07fa662016-05-23 11:32:43 -07001872 u8 epnum = (i << 1) | (direction ? 1 : 0);
Felipe Balbi72246da2011-08-19 18:10:58 +03001873
Felipe Balbi72246da2011-08-19 18:10:58 +03001874 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
Jingoo Han734d5a52014-07-17 12:45:11 +09001875 if (!dep)
Felipe Balbi72246da2011-08-19 18:10:58 +03001876 return -ENOMEM;
Felipe Balbi72246da2011-08-19 18:10:58 +03001877
1878 dep->dwc = dwc;
1879 dep->number = epnum;
Felipe Balbi9aa62ae2013-07-12 19:10:59 +03001880 dep->direction = !!direction;
Felipe Balbi2eb88012016-04-12 16:53:39 +03001881 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
Felipe Balbi72246da2011-08-19 18:10:58 +03001882 dwc->eps[epnum] = dep;
1883
1884 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1885 (epnum & 1) ? "in" : "out");
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001886
Felipe Balbi72246da2011-08-19 18:10:58 +03001887 dep->endpoint.name = dep->name;
Felipe Balbi74674cb2016-04-13 16:44:39 +03001888 spin_lock_init(&dep->lock);
Felipe Balbi72246da2011-08-19 18:10:58 +03001889
1890 if (epnum == 0 || epnum == 1) {
Robert Baldygae117e742013-12-13 12:23:38 +01001891 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
Pratyush Anand6048e4c2013-01-18 16:53:56 +05301892 dep->endpoint.maxburst = 1;
Felipe Balbi72246da2011-08-19 18:10:58 +03001893 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1894 if (!epnum)
1895 dwc->gadget.ep0 = &dep->endpoint;
1896 } else {
1897 int ret;
1898
Robert Baldygae117e742013-12-13 12:23:38 +01001899 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
Sebastian Andrzej Siewior12d36c12011-11-03 20:27:50 +01001900 dep->endpoint.max_streams = 15;
Felipe Balbi72246da2011-08-19 18:10:58 +03001901 dep->endpoint.ops = &dwc3_gadget_ep_ops;
1902 list_add_tail(&dep->endpoint.ep_list,
1903 &dwc->gadget.ep_list);
1904
1905 ret = dwc3_alloc_trb_pool(dep);
Felipe Balbi25b8ff62011-11-04 12:32:47 +02001906 if (ret)
Felipe Balbi72246da2011-08-19 18:10:58 +03001907 return ret;
Felipe Balbi72246da2011-08-19 18:10:58 +03001908 }
Felipe Balbi25b8ff62011-11-04 12:32:47 +02001909
Robert Baldygaa474d3b2015-07-31 16:00:19 +02001910 if (epnum == 0 || epnum == 1) {
1911 dep->endpoint.caps.type_control = true;
1912 } else {
1913 dep->endpoint.caps.type_iso = true;
1914 dep->endpoint.caps.type_bulk = true;
1915 dep->endpoint.caps.type_int = true;
1916 }
1917
1918 dep->endpoint.caps.dir_in = !!direction;
1919 dep->endpoint.caps.dir_out = !direction;
1920
Felipe Balbiaa3342c2016-03-14 11:01:31 +02001921 INIT_LIST_HEAD(&dep->pending_list);
1922 INIT_LIST_HEAD(&dep->started_list);
Felipe Balbi72246da2011-08-19 18:10:58 +03001923 }
1924
1925 return 0;
1926}
1927
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001928static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1929{
1930 int ret;
1931
1932 INIT_LIST_HEAD(&dwc->gadget.ep_list);
1933
1934 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1935 if (ret < 0) {
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02001936 dev_err(dwc->dev, "failed to initialize OUT endpoints\n");
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001937 return ret;
1938 }
1939
1940 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1941 if (ret < 0) {
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02001942 dev_err(dwc->dev, "failed to initialize IN endpoints\n");
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001943 return ret;
1944 }
1945
1946 return 0;
1947}
1948
Felipe Balbi72246da2011-08-19 18:10:58 +03001949static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1950{
1951 struct dwc3_ep *dep;
1952 u8 epnum;
1953
1954 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1955 dep = dwc->eps[epnum];
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03001956 if (!dep)
1957 continue;
George Cherian5bf8fae2013-05-27 14:35:49 +05301958 /*
1959 * Physical endpoints 0 and 1 are special; they form the
1960 * bi-directional USB endpoint 0.
1961 *
1962 * For those two physical endpoints, we don't allocate a TRB
1963 * pool nor do we add them the endpoints list. Due to that, we
1964 * shouldn't do these two operations otherwise we would end up
1965 * with all sorts of bugs when removing dwc3.ko.
1966 */
1967 if (epnum != 0 && epnum != 1) {
1968 dwc3_free_trb_pool(dep);
Felipe Balbi72246da2011-08-19 18:10:58 +03001969 list_del(&dep->endpoint.ep_list);
George Cherian5bf8fae2013-05-27 14:35:49 +05301970 }
Felipe Balbi72246da2011-08-19 18:10:58 +03001971
1972 kfree(dep);
1973 }
1974}
1975
Felipe Balbi72246da2011-08-19 18:10:58 +03001976/* -------------------------------------------------------------------------- */
Felipe Balbie5caff62013-02-26 15:11:05 +02001977
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05301978static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1979 struct dwc3_request *req, struct dwc3_trb *trb,
Felipe Balbie5b36ae2016-08-10 11:13:26 +03001980 const struct dwc3_event_depevt *event, int status,
1981 int chain)
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05301982{
1983 unsigned int count;
1984 unsigned int s_pkt = 0;
1985 unsigned int trb_status;
1986
Felipe Balbidc55c672016-08-12 13:20:32 +03001987 dwc3_ep_inc_deq(dep);
Felipe Balbia9c3ca52016-10-05 14:24:37 +03001988
1989 if (req->trb == trb)
1990 dep->queued_requests--;
1991
Felipe Balbi2c4cbe6e52014-04-30 17:45:10 -05001992 trace_dwc3_complete_trb(dep, trb);
1993
Felipe Balbie5b36ae2016-08-10 11:13:26 +03001994 /*
1995 * If we're in the middle of series of chained TRBs and we
1996 * receive a short transfer along the way, DWC3 will skip
1997 * through all TRBs including the last TRB in the chain (the
1998 * where CHN bit is zero. DWC3 will also avoid clearing HWO
1999 * bit and SW has to do it manually.
2000 *
2001 * We're going to do that here to avoid problems of HW trying
2002 * to use bogus TRBs for transfers.
2003 */
2004 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2005 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2006
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302007 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
Felipe Balbia0ad85a2016-08-10 18:07:46 +03002008 return 1;
Felipe Balbie5b36ae2016-08-10 11:13:26 +03002009
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302010 count = trb->size & DWC3_TRB_SIZE_MASK;
Felipe Balbie62c5bc52016-10-25 13:47:21 +03002011 req->remaining += count;
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302012
2013 if (dep->direction) {
2014 if (count) {
2015 trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
2016 if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302017 /*
2018 * If missed isoc occurred and there is
2019 * no request queued then issue END
2020 * TRANSFER, so that core generates
2021 * next xfernotready and we will issue
2022 * a fresh START TRANSFER.
2023 * If there are still queued request
2024 * then wait, do not issue either END
2025 * or UPDATE TRANSFER, just attach next
Felipe Balbiaa3342c2016-03-14 11:01:31 +02002026 * request in pending_list during
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302027 * giveback.If any future queued request
2028 * is successfully transferred then we
2029 * will issue UPDATE TRANSFER for all
Felipe Balbiaa3342c2016-03-14 11:01:31 +02002030 * request in the pending_list.
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302031 */
2032 dep->flags |= DWC3_EP_MISSED_ISOC;
2033 } else {
2034 dev_err(dwc->dev, "incomplete IN transfer %s\n",
2035 dep->name);
2036 status = -ECONNRESET;
2037 }
2038 } else {
2039 dep->flags &= ~DWC3_EP_MISSED_ISOC;
2040 }
2041 } else {
2042 if (count && (event->status & DEPEVT_STATUS_SHORT))
2043 s_pkt = 1;
2044 }
2045
Felipe Balbi7c705df2016-08-10 12:35:30 +03002046 if (s_pkt && !chain)
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302047 return 1;
Felipe Balbif99f53f2016-08-12 13:19:20 +03002048
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302049 if ((event->status & DEPEVT_STATUS_IOC) &&
2050 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2051 return 1;
Felipe Balbif99f53f2016-08-12 13:19:20 +03002052
Pratyush Anande5ba5ec2013-01-14 15:59:37 +05302053 return 0;
2054}
2055
Felipe Balbi72246da2011-08-19 18:10:58 +03002056static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
2057 const struct dwc3_event_depevt *event, int status)
2058{
Felipe Balbi31162af2016-08-11 14:38:37 +03002059 struct dwc3_request *req, *n;
Felipe Balbif6bafc62012-02-06 11:04:53 +02002060 struct dwc3_trb *trb;
Arnd Bergmannd6e10bf2016-09-09 12:01:51 +02002061 bool ioc = false;
Felipe Balbie62c5bc52016-10-25 13:47:21 +03002062 int ret = 0;
Felipe Balbi72246da2011-08-19 18:10:58 +03002063
Felipe Balbi31162af2016-08-11 14:38:37 +03002064 list_for_each_entry_safe(req, n, &dep->started_list, list) {
Felipe Balbi1f512112016-08-12 13:17:27 +03002065 unsigned length;
Felipe Balbie5b36ae2016-08-10 11:13:26 +03002066 int chain;
2067
Felipe Balbi1f512112016-08-12 13:17:27 +03002068 length = req->request.length;
2069 chain = req->num_pending_sgs > 0;
Felipe Balbi31162af2016-08-11 14:38:37 +03002070 if (chain) {
Felipe Balbi1f512112016-08-12 13:17:27 +03002071 struct scatterlist *sg = req->sg;
Felipe Balbi31162af2016-08-11 14:38:37 +03002072 struct scatterlist *s;
Felipe Balbi1f512112016-08-12 13:17:27 +03002073 unsigned int pending = req->num_pending_sgs;
Felipe Balbi31162af2016-08-11 14:38:37 +03002074 unsigned int i;
Felipe Balbiac7bdcc2015-11-16 16:13:57 -06002075
Felipe Balbi1f512112016-08-12 13:17:27 +03002076 for_each_sg(sg, s, pending, i) {
Felipe Balbi31162af2016-08-11 14:38:37 +03002077 trb = &dep->trb_pool[dep->trb_dequeue];
Felipe Balbic7de5732016-07-29 03:17:58 +03002078
Felipe Balbi7282c4e2016-10-25 13:50:46 +03002079 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2080 break;
2081
Felipe Balbi1f512112016-08-12 13:17:27 +03002082 req->sg = sg_next(s);
2083 req->num_pending_sgs--;
2084
Felipe Balbi31162af2016-08-11 14:38:37 +03002085 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2086 event, status, chain);
Felipe Balbi1f512112016-08-12 13:17:27 +03002087 if (ret)
2088 break;
Felipe Balbi31162af2016-08-11 14:38:37 +03002089 }
2090 } else {
Felipe Balbi737f1ae2016-08-11 12:24:27 +03002091 trb = &dep->trb_pool[dep->trb_dequeue];
Ville Syrjäläd115d702015-08-31 19:48:28 +03002092 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
Felipe Balbie5b36ae2016-08-10 11:13:26 +03002093 event, status, chain);
Felipe Balbi31162af2016-08-11 14:38:37 +03002094 }
Ville Syrjäläd115d702015-08-31 19:48:28 +03002095
Felipe Balbie62c5bc52016-10-25 13:47:21 +03002096 req->request.actual = length - req->remaining;
Felipe Balbi1f512112016-08-12 13:17:27 +03002097
Felipe Balbiff377ae2016-10-25 13:54:00 +03002098 if ((req->request.actual < length) && req->num_pending_sgs)
Felipe Balbi1f512112016-08-12 13:17:27 +03002099 return __dwc3_gadget_kick_transfer(dep, 0);
2100
Ville Syrjäläd115d702015-08-31 19:48:28 +03002101 dwc3_gadget_giveback(dep, req, status);
2102
Arnd Bergmannd6e10bf2016-09-09 12:01:51 +02002103 if (ret) {
2104 if ((event->status & DEPEVT_STATUS_IOC) &&
2105 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2106 ioc = true;
Felipe Balbi72246da2011-08-19 18:10:58 +03002107 break;
Arnd Bergmannd6e10bf2016-09-09 12:01:51 +02002108 }
Felipe Balbi31162af2016-08-11 14:38:37 +03002109 }
Felipe Balbi72246da2011-08-19 18:10:58 +03002110
Felipe Balbi4cb42212016-05-18 12:37:21 +03002111 /*
2112 * Our endpoint might get disabled by another thread during
2113 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2114 * early on so DWC3_EP_BUSY flag gets cleared
2115 */
2116 if (!dep->endpoint.desc)
2117 return 1;
2118
Pratyush Anandcdc359d2013-01-14 15:59:34 +05302119 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
Felipe Balbiaa3342c2016-03-14 11:01:31 +02002120 list_empty(&dep->started_list)) {
2121 if (list_empty(&dep->pending_list)) {
Pratyush Anandcdc359d2013-01-14 15:59:34 +05302122 /*
2123 * If there is no entry in request list then do
2124 * not issue END TRANSFER now. Just set PENDING
2125 * flag, so that END TRANSFER is issued when an
2126 * entry is added into request list.
2127 */
2128 dep->flags = DWC3_EP_PENDING_REQUEST;
2129 } else {
Paul Zimmermanb992e682012-04-27 14:17:35 +03002130 dwc3_stop_active_transfer(dwc, dep->number, true);
Pratyush Anandcdc359d2013-01-14 15:59:34 +05302131 dep->flags = DWC3_EP_ENABLED;
2132 }
Pratyush Anand7efea862013-01-14 15:59:32 +05302133 return 1;
2134 }
2135
Arnd Bergmannd6e10bf2016-09-09 12:01:51 +02002136 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && ioc)
2137 return 0;
2138
Felipe Balbi72246da2011-08-19 18:10:58 +03002139 return 1;
2140}
2141
2142static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
Jingoo Han029d97f2014-07-04 15:00:51 +09002143 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
Felipe Balbi72246da2011-08-19 18:10:58 +03002144{
2145 unsigned status = 0;
2146 int clean_busy;
Felipe Balbie18b7972015-05-29 10:06:38 -05002147 u32 is_xfer_complete;
2148
2149 is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
Felipe Balbi72246da2011-08-19 18:10:58 +03002150
2151 if (event->status & DEPEVT_STATUS_BUSERR)
2152 status = -ECONNRESET;
2153
Paul Zimmerman1d046792012-02-15 18:56:56 -08002154 clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
Felipe Balbi4cb42212016-05-18 12:37:21 +03002155 if (clean_busy && (!dep->endpoint.desc || is_xfer_complete ||
Felipe Balbie18b7972015-05-29 10:06:38 -05002156 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
Felipe Balbi72246da2011-08-19 18:10:58 +03002157 dep->flags &= ~DWC3_EP_BUSY;
Felipe Balbifae2b902011-10-14 13:00:30 +03002158
2159 /*
2160 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2161 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2162 */
2163 if (dwc->revision < DWC3_REVISION_183A) {
2164 u32 reg;
2165 int i;
2166
2167 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
Moiz Sonasath348e0262012-08-01 14:08:30 -05002168 dep = dwc->eps[i];
Felipe Balbifae2b902011-10-14 13:00:30 +03002169
2170 if (!(dep->flags & DWC3_EP_ENABLED))
2171 continue;
2172
Felipe Balbiaa3342c2016-03-14 11:01:31 +02002173 if (!list_empty(&dep->started_list))
Felipe Balbifae2b902011-10-14 13:00:30 +03002174 return;
2175 }
2176
2177 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2178 reg |= dwc->u1u2;
2179 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2180
2181 dwc->u1u2 = 0;
2182 }
Felipe Balbi8a1a9c92015-09-21 14:32:00 -05002183
Felipe Balbi4cb42212016-05-18 12:37:21 +03002184 /*
2185 * Our endpoint might get disabled by another thread during
2186 * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2187 * early on so DWC3_EP_BUSY flag gets cleared
2188 */
2189 if (!dep->endpoint.desc)
2190 return;
2191
Felipe Balbie6e709b2015-09-28 15:16:56 -05002192 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
Felipe Balbi8a1a9c92015-09-21 14:32:00 -05002193 int ret;
2194
Felipe Balbi4fae2e32016-05-12 16:53:59 +03002195 ret = __dwc3_gadget_kick_transfer(dep, 0);
Felipe Balbi8a1a9c92015-09-21 14:32:00 -05002196 if (!ret || ret == -EBUSY)
2197 return;
2198 }
Felipe Balbi72246da2011-08-19 18:10:58 +03002199}
2200
Felipe Balbi72246da2011-08-19 18:10:58 +03002201static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2202 const struct dwc3_event_depevt *event)
2203{
2204 struct dwc3_ep *dep;
2205 u8 epnum = event->endpoint_number;
Baolin Wang76a638f2016-10-31 19:38:36 +08002206 u8 cmd;
Felipe Balbi72246da2011-08-19 18:10:58 +03002207
2208 dep = dwc->eps[epnum];
2209
Baolin Wang76a638f2016-10-31 19:38:36 +08002210 if (!(dep->flags & DWC3_EP_ENABLED) &&
2211 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
Felipe Balbi3336abb2012-06-06 09:19:35 +03002212 return;
2213
Felipe Balbi72246da2011-08-19 18:10:58 +03002214 if (epnum == 0 || epnum == 1) {
2215 dwc3_ep0_interrupt(dwc, event);
2216 return;
2217 }
2218
2219 switch (event->endpoint_event) {
2220 case DWC3_DEPEVT_XFERCOMPLETE:
Felipe Balbib4996a82012-06-06 12:04:13 +03002221 dep->resource_index = 0;
Paul Zimmermanc2df85c2012-02-24 17:32:18 -08002222
Ido Shayevitz16e78db2012-03-12 20:25:24 +02002223 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
Felipe Balbi8566cd12016-09-26 11:16:39 +03002224 dev_err(dwc->dev, "XferComplete for Isochronous endpoint\n");
Felipe Balbi72246da2011-08-19 18:10:58 +03002225 return;
2226 }
2227
Jingoo Han029d97f2014-07-04 15:00:51 +09002228 dwc3_endpoint_transfer_complete(dwc, dep, event);
Felipe Balbi72246da2011-08-19 18:10:58 +03002229 break;
2230 case DWC3_DEPEVT_XFERINPROGRESS:
Jingoo Han029d97f2014-07-04 15:00:51 +09002231 dwc3_endpoint_transfer_complete(dwc, dep, event);
Felipe Balbi72246da2011-08-19 18:10:58 +03002232 break;
2233 case DWC3_DEPEVT_XFERNOTREADY:
Ido Shayevitz16e78db2012-03-12 20:25:24 +02002234 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
Felipe Balbi72246da2011-08-19 18:10:58 +03002235 dwc3_gadget_start_isoc(dwc, dep, event);
2236 } else {
2237 int ret;
2238
Felipe Balbi4fae2e32016-05-12 16:53:59 +03002239 ret = __dwc3_gadget_kick_transfer(dep, 0);
Felipe Balbi72246da2011-08-19 18:10:58 +03002240 if (!ret || ret == -EBUSY)
2241 return;
Felipe Balbi72246da2011-08-19 18:10:58 +03002242 }
2243
2244 break;
Felipe Balbi879631a2011-09-30 10:58:47 +03002245 case DWC3_DEPEVT_STREAMEVT:
Ido Shayevitz16e78db2012-03-12 20:25:24 +02002246 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
Felipe Balbi879631a2011-09-30 10:58:47 +03002247 dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2248 dep->name);
2249 return;
2250 }
Felipe Balbi879631a2011-09-30 10:58:47 +03002251 break;
Felipe Balbi72246da2011-08-19 18:10:58 +03002252 case DWC3_DEPEVT_EPCMDCMPLT:
Baolin Wang76a638f2016-10-31 19:38:36 +08002253 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2254
2255 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2256 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2257 wake_up(&dep->wait_end_transfer);
2258 }
2259 break;
2260 case DWC3_DEPEVT_RXTXFIFOEVT:
Felipe Balbi72246da2011-08-19 18:10:58 +03002261 break;
2262 }
2263}
2264
2265static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2266{
2267 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2268 spin_unlock(&dwc->lock);
2269 dwc->gadget_driver->disconnect(&dwc->gadget);
2270 spin_lock(&dwc->lock);
2271 }
2272}
2273
Felipe Balbibc5ba2e2014-02-26 10:17:07 -06002274static void dwc3_suspend_gadget(struct dwc3 *dwc)
2275{
Dan Carpenter73a30bf2014-03-07 14:19:57 +03002276 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
Felipe Balbibc5ba2e2014-02-26 10:17:07 -06002277 spin_unlock(&dwc->lock);
2278 dwc->gadget_driver->suspend(&dwc->gadget);
2279 spin_lock(&dwc->lock);
2280 }
2281}
2282
2283static void dwc3_resume_gadget(struct dwc3 *dwc)
2284{
Dan Carpenter73a30bf2014-03-07 14:19:57 +03002285 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
Felipe Balbibc5ba2e2014-02-26 10:17:07 -06002286 spin_unlock(&dwc->lock);
2287 dwc->gadget_driver->resume(&dwc->gadget);
Felipe Balbi5c7b3b02015-01-29 10:29:18 -06002288 spin_lock(&dwc->lock);
Felipe Balbi8e744752014-11-06 14:27:53 +08002289 }
2290}
2291
2292static void dwc3_reset_gadget(struct dwc3 *dwc)
2293{
2294 if (!dwc->gadget_driver)
2295 return;
2296
2297 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2298 spin_unlock(&dwc->lock);
2299 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
Felipe Balbibc5ba2e2014-02-26 10:17:07 -06002300 spin_lock(&dwc->lock);
2301 }
2302}
2303
Paul Zimmermanb992e682012-04-27 14:17:35 +03002304static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
Felipe Balbi72246da2011-08-19 18:10:58 +03002305{
2306 struct dwc3_ep *dep;
2307 struct dwc3_gadget_ep_cmd_params params;
2308 u32 cmd;
2309 int ret;
2310
2311 dep = dwc->eps[epnum];
2312
Baolin Wang76a638f2016-10-31 19:38:36 +08002313 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2314 !dep->resource_index)
Pratyush Anand3daf74d2012-06-23 02:23:08 +05302315 return;
2316
Pratyush Anand57911502012-07-06 15:19:10 +05302317 /*
2318 * NOTICE: We are violating what the Databook says about the
2319 * EndTransfer command. Ideally we would _always_ wait for the
2320 * EndTransfer Command Completion IRQ, but that's causing too
2321 * much trouble synchronizing between us and gadget driver.
2322 *
2323 * We have discussed this with the IP Provider and it was
2324 * suggested to giveback all requests here, but give HW some
2325 * extra time to synchronize with the interconnect. We're using
Mickael Maisondc93b412014-12-23 17:34:43 +01002326 * an arbitrary 100us delay for that.
Pratyush Anand57911502012-07-06 15:19:10 +05302327 *
2328 * Note also that a similar handling was tested by Synopsys
2329 * (thanks a lot Paul) and nothing bad has come out of it.
2330 * In short, what we're doing is:
2331 *
2332 * - Issue EndTransfer WITH CMDIOC bit set
2333 * - Wait 100us
John Youn06281d42016-08-22 15:39:13 -07002334 *
2335 * As of IP version 3.10a of the DWC_usb3 IP, the controller
2336 * supports a mode to work around the above limitation. The
2337 * software can poll the CMDACT bit in the DEPCMD register
2338 * after issuing a EndTransfer command. This mode is enabled
2339 * by writing GUCTL2[14]. This polling is already done in the
2340 * dwc3_send_gadget_ep_cmd() function so if the mode is
2341 * enabled, the EndTransfer command will have completed upon
2342 * returning from this function and we don't need to delay for
2343 * 100us.
2344 *
2345 * This mode is NOT available on the DWC_usb31 IP.
Pratyush Anand57911502012-07-06 15:19:10 +05302346 */
2347
Pratyush Anand3daf74d2012-06-23 02:23:08 +05302348 cmd = DWC3_DEPCMD_ENDTRANSFER;
Paul Zimmermanb992e682012-04-27 14:17:35 +03002349 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2350 cmd |= DWC3_DEPCMD_CMDIOC;
Felipe Balbib4996a82012-06-06 12:04:13 +03002351 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
Pratyush Anand3daf74d2012-06-23 02:23:08 +05302352 memset(&params, 0, sizeof(params));
Felipe Balbi2cd47182016-04-12 16:42:43 +03002353 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
Pratyush Anand3daf74d2012-06-23 02:23:08 +05302354 WARN_ON_ONCE(ret);
Felipe Balbib4996a82012-06-06 12:04:13 +03002355 dep->resource_index = 0;
Felipe Balbi041d81f2012-10-04 11:58:00 +03002356 dep->flags &= ~DWC3_EP_BUSY;
John Youn06281d42016-08-22 15:39:13 -07002357
Baolin Wang76a638f2016-10-31 19:38:36 +08002358 if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2359 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
John Youn06281d42016-08-22 15:39:13 -07002360 udelay(100);
Baolin Wang76a638f2016-10-31 19:38:36 +08002361 }
Felipe Balbi72246da2011-08-19 18:10:58 +03002362}
2363
Felipe Balbi72246da2011-08-19 18:10:58 +03002364static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2365{
2366 u32 epnum;
2367
2368 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2369 struct dwc3_ep *dep;
Felipe Balbi72246da2011-08-19 18:10:58 +03002370 int ret;
2371
2372 dep = dwc->eps[epnum];
Felipe Balbi6a1e3ef2011-05-05 16:21:59 +03002373 if (!dep)
2374 continue;
Felipe Balbi72246da2011-08-19 18:10:58 +03002375
2376 if (!(dep->flags & DWC3_EP_STALL))
2377 continue;
2378
2379 dep->flags &= ~DWC3_EP_STALL;
2380
John Youn50c763f2016-05-31 17:49:56 -07002381 ret = dwc3_send_clear_stall_ep_cmd(dep);
Felipe Balbi72246da2011-08-19 18:10:58 +03002382 WARN_ON_ONCE(ret);
2383 }
2384}
2385
2386static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2387{
Felipe Balbic4430a22012-05-24 10:30:01 +03002388 int reg;
2389
Felipe Balbi72246da2011-08-19 18:10:58 +03002390 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2391 reg &= ~DWC3_DCTL_INITU1ENA;
2392 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2393
2394 reg &= ~DWC3_DCTL_INITU2ENA;
2395 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
Felipe Balbi72246da2011-08-19 18:10:58 +03002396
Felipe Balbi72246da2011-08-19 18:10:58 +03002397 dwc3_disconnect_gadget(dwc);
2398
2399 dwc->gadget.speed = USB_SPEED_UNKNOWN;
Felipe Balbidf62df52011-10-14 15:11:49 +03002400 dwc->setup_packet_pending = false;
Felipe Balbi06a374e2014-10-10 15:24:00 -05002401 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
Felipe Balbifc8bb912016-05-16 13:14:48 +03002402
2403 dwc->connected = false;
Felipe Balbi72246da2011-08-19 18:10:58 +03002404}
2405
Felipe Balbi72246da2011-08-19 18:10:58 +03002406static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2407{
2408 u32 reg;
2409
Felipe Balbifc8bb912016-05-16 13:14:48 +03002410 dwc->connected = true;
2411
Felipe Balbidf62df52011-10-14 15:11:49 +03002412 /*
2413 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2414 * would cause a missing Disconnect Event if there's a
2415 * pending Setup Packet in the FIFO.
2416 *
2417 * There's no suggested workaround on the official Bug
2418 * report, which states that "unless the driver/application
2419 * is doing any special handling of a disconnect event,
2420 * there is no functional issue".
2421 *
2422 * Unfortunately, it turns out that we _do_ some special
2423 * handling of a disconnect event, namely complete all
2424 * pending transfers, notify gadget driver of the
2425 * disconnection, and so on.
2426 *
2427 * Our suggested workaround is to follow the Disconnect
2428 * Event steps here, instead, based on a setup_packet_pending
Felipe Balbib5d335e2015-11-16 16:20:34 -06002429 * flag. Such flag gets set whenever we have a SETUP_PENDING
2430 * status for EP0 TRBs and gets cleared on XferComplete for the
Felipe Balbidf62df52011-10-14 15:11:49 +03002431 * same endpoint.
2432 *
2433 * Refers to:
2434 *
2435 * STAR#9000466709: RTL: Device : Disconnect event not
2436 * generated if setup packet pending in FIFO
2437 */
2438 if (dwc->revision < DWC3_REVISION_188A) {
2439 if (dwc->setup_packet_pending)
2440 dwc3_gadget_disconnect_interrupt(dwc);
2441 }
2442
Felipe Balbi8e744752014-11-06 14:27:53 +08002443 dwc3_reset_gadget(dwc);
Felipe Balbi72246da2011-08-19 18:10:58 +03002444
2445 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2446 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2447 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
Gerard Cauvy3b637362012-02-10 12:21:18 +02002448 dwc->test_mode = false;
Felipe Balbi72246da2011-08-19 18:10:58 +03002449 dwc3_clear_stall_all_ep(dwc);
2450
2451 /* Reset device address to zero */
2452 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2453 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2454 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
Felipe Balbi72246da2011-08-19 18:10:58 +03002455}
2456
2457static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2458{
2459 u32 reg;
2460 u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2461
2462 /*
2463 * We change the clock only at SS but I dunno why I would want to do
2464 * this. Maybe it becomes part of the power saving plan.
2465 */
2466
John Younee5cd412016-02-05 17:08:45 -08002467 if ((speed != DWC3_DSTS_SUPERSPEED) &&
2468 (speed != DWC3_DSTS_SUPERSPEED_PLUS))
Felipe Balbi72246da2011-08-19 18:10:58 +03002469 return;
2470
2471 /*
2472 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2473 * each time on Connect Done.
2474 */
2475 if (!usb30_clock)
2476 return;
2477
2478 reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2479 reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2480 dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2481}
2482
Felipe Balbi72246da2011-08-19 18:10:58 +03002483static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2484{
Felipe Balbi72246da2011-08-19 18:10:58 +03002485 struct dwc3_ep *dep;
2486 int ret;
2487 u32 reg;
2488 u8 speed;
2489
Felipe Balbi72246da2011-08-19 18:10:58 +03002490 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2491 speed = reg & DWC3_DSTS_CONNECTSPD;
2492 dwc->speed = speed;
2493
2494 dwc3_update_ram_clk_sel(dwc, speed);
2495
2496 switch (speed) {
John Youn2da9ad72016-05-20 16:34:26 -07002497 case DWC3_DSTS_SUPERSPEED_PLUS:
John Youn75808622016-02-05 17:09:13 -08002498 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2499 dwc->gadget.ep0->maxpacket = 512;
2500 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2501 break;
John Youn2da9ad72016-05-20 16:34:26 -07002502 case DWC3_DSTS_SUPERSPEED:
Felipe Balbi05870c52011-10-14 14:51:38 +03002503 /*
2504 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2505 * would cause a missing USB3 Reset event.
2506 *
2507 * In such situations, we should force a USB3 Reset
2508 * event by calling our dwc3_gadget_reset_interrupt()
2509 * routine.
2510 *
2511 * Refers to:
2512 *
2513 * STAR#9000483510: RTL: SS : USB3 reset event may
2514 * not be generated always when the link enters poll
2515 */
2516 if (dwc->revision < DWC3_REVISION_190A)
2517 dwc3_gadget_reset_interrupt(dwc);
2518
Felipe Balbi72246da2011-08-19 18:10:58 +03002519 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2520 dwc->gadget.ep0->maxpacket = 512;
2521 dwc->gadget.speed = USB_SPEED_SUPER;
2522 break;
John Youn2da9ad72016-05-20 16:34:26 -07002523 case DWC3_DSTS_HIGHSPEED:
Felipe Balbi72246da2011-08-19 18:10:58 +03002524 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2525 dwc->gadget.ep0->maxpacket = 64;
2526 dwc->gadget.speed = USB_SPEED_HIGH;
2527 break;
John Youn2da9ad72016-05-20 16:34:26 -07002528 case DWC3_DSTS_FULLSPEED2:
2529 case DWC3_DSTS_FULLSPEED1:
Felipe Balbi72246da2011-08-19 18:10:58 +03002530 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2531 dwc->gadget.ep0->maxpacket = 64;
2532 dwc->gadget.speed = USB_SPEED_FULL;
2533 break;
John Youn2da9ad72016-05-20 16:34:26 -07002534 case DWC3_DSTS_LOWSPEED:
Felipe Balbi72246da2011-08-19 18:10:58 +03002535 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2536 dwc->gadget.ep0->maxpacket = 8;
2537 dwc->gadget.speed = USB_SPEED_LOW;
2538 break;
2539 }
2540
Pratyush Anand2b758352013-01-14 15:59:31 +05302541 /* Enable USB2 LPM Capability */
2542
John Younee5cd412016-02-05 17:08:45 -08002543 if ((dwc->revision > DWC3_REVISION_194A) &&
John Youn2da9ad72016-05-20 16:34:26 -07002544 (speed != DWC3_DSTS_SUPERSPEED) &&
2545 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
Pratyush Anand2b758352013-01-14 15:59:31 +05302546 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2547 reg |= DWC3_DCFG_LPM_CAP;
2548 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2549
2550 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2551 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2552
Huang Rui460d0982014-10-31 11:11:18 +08002553 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
Pratyush Anand2b758352013-01-14 15:59:31 +05302554
Huang Rui80caf7d2014-10-28 19:54:26 +08002555 /*
2556 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2557 * DCFG.LPMCap is set, core responses with an ACK and the
2558 * BESL value in the LPM token is less than or equal to LPM
2559 * NYET threshold.
2560 */
2561 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2562 && dwc->has_lpm_erratum,
2563 "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2564
2565 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2566 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2567
Pratyush Anand2b758352013-01-14 15:59:31 +05302568 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
Felipe Balbi356363b2013-12-19 16:37:05 -06002569 } else {
2570 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2571 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2572 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
Pratyush Anand2b758352013-01-14 15:59:31 +05302573 }
2574
Felipe Balbi72246da2011-08-19 18:10:58 +03002575 dep = dwc->eps[0];
Paul Zimmerman265b70a2013-12-19 12:38:49 -06002576 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2577 false);
Felipe Balbi72246da2011-08-19 18:10:58 +03002578 if (ret) {
2579 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2580 return;
2581 }
2582
2583 dep = dwc->eps[1];
Paul Zimmerman265b70a2013-12-19 12:38:49 -06002584 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2585 false);
Felipe Balbi72246da2011-08-19 18:10:58 +03002586 if (ret) {
2587 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2588 return;
2589 }
2590
2591 /*
2592 * Configure PHY via GUSB3PIPECTLn if required.
2593 *
2594 * Update GTXFIFOSIZn
2595 *
2596 * In both cases reset values should be sufficient.
2597 */
2598}
2599
2600static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2601{
Felipe Balbi72246da2011-08-19 18:10:58 +03002602 /*
2603 * TODO take core out of low power mode when that's
2604 * implemented.
2605 */
2606
Jiebing Liad14d4e2014-12-11 13:26:29 +08002607 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2608 spin_unlock(&dwc->lock);
2609 dwc->gadget_driver->resume(&dwc->gadget);
2610 spin_lock(&dwc->lock);
2611 }
Felipe Balbi72246da2011-08-19 18:10:58 +03002612}
2613
2614static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2615 unsigned int evtinfo)
2616{
Felipe Balbifae2b902011-10-14 13:00:30 +03002617 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
Felipe Balbi0b0cc1c2012-09-18 21:39:24 +03002618 unsigned int pwropt;
2619
2620 /*
2621 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2622 * Hibernation mode enabled which would show up when device detects
2623 * host-initiated U3 exit.
2624 *
2625 * In that case, device will generate a Link State Change Interrupt
2626 * from U3 to RESUME which is only necessary if Hibernation is
2627 * configured in.
2628 *
2629 * There are no functional changes due to such spurious event and we
2630 * just need to ignore it.
2631 *
2632 * Refers to:
2633 *
2634 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2635 * operational mode
2636 */
2637 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2638 if ((dwc->revision < DWC3_REVISION_250A) &&
2639 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2640 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2641 (next == DWC3_LINK_STATE_RESUME)) {
Felipe Balbi0b0cc1c2012-09-18 21:39:24 +03002642 return;
2643 }
2644 }
Felipe Balbifae2b902011-10-14 13:00:30 +03002645
2646 /*
2647 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2648 * on the link partner, the USB session might do multiple entry/exit
2649 * of low power states before a transfer takes place.
2650 *
2651 * Due to this problem, we might experience lower throughput. The
2652 * suggested workaround is to disable DCTL[12:9] bits if we're
2653 * transitioning from U1/U2 to U0 and enable those bits again
2654 * after a transfer completes and there are no pending transfers
2655 * on any of the enabled endpoints.
2656 *
2657 * This is the first half of that workaround.
2658 *
2659 * Refers to:
2660 *
2661 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2662 * core send LGO_Ux entering U0
2663 */
2664 if (dwc->revision < DWC3_REVISION_183A) {
2665 if (next == DWC3_LINK_STATE_U0) {
2666 u32 u1u2;
2667 u32 reg;
2668
2669 switch (dwc->link_state) {
2670 case DWC3_LINK_STATE_U1:
2671 case DWC3_LINK_STATE_U2:
2672 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2673 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2674 | DWC3_DCTL_ACCEPTU2ENA
2675 | DWC3_DCTL_INITU1ENA
2676 | DWC3_DCTL_ACCEPTU1ENA);
2677
2678 if (!dwc->u1u2)
2679 dwc->u1u2 = reg & u1u2;
2680
2681 reg &= ~u1u2;
2682
2683 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2684 break;
2685 default:
2686 /* do nothing */
2687 break;
2688 }
2689 }
2690 }
2691
Felipe Balbibc5ba2e2014-02-26 10:17:07 -06002692 switch (next) {
2693 case DWC3_LINK_STATE_U1:
2694 if (dwc->speed == USB_SPEED_SUPER)
2695 dwc3_suspend_gadget(dwc);
2696 break;
2697 case DWC3_LINK_STATE_U2:
2698 case DWC3_LINK_STATE_U3:
2699 dwc3_suspend_gadget(dwc);
2700 break;
2701 case DWC3_LINK_STATE_RESUME:
2702 dwc3_resume_gadget(dwc);
2703 break;
2704 default:
2705 /* do nothing */
2706 break;
2707 }
2708
Felipe Balbie57ebc12014-04-22 13:20:12 -05002709 dwc->link_state = next;
Felipe Balbi72246da2011-08-19 18:10:58 +03002710}
2711
Baolin Wang72704f82016-05-16 16:43:53 +08002712static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
2713 unsigned int evtinfo)
2714{
2715 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2716
2717 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
2718 dwc3_suspend_gadget(dwc);
2719
2720 dwc->link_state = next;
2721}
2722
Felipe Balbie1dadd32014-02-25 14:47:54 -06002723static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2724 unsigned int evtinfo)
2725{
2726 unsigned int is_ss = evtinfo & BIT(4);
2727
2728 /**
2729 * WORKAROUND: DWC3 revison 2.20a with hibernation support
2730 * have a known issue which can cause USB CV TD.9.23 to fail
2731 * randomly.
2732 *
2733 * Because of this issue, core could generate bogus hibernation
2734 * events which SW needs to ignore.
2735 *
2736 * Refers to:
2737 *
2738 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2739 * Device Fallback from SuperSpeed
2740 */
2741 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2742 return;
2743
2744 /* enter hibernation here */
2745}
2746
Felipe Balbi72246da2011-08-19 18:10:58 +03002747static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2748 const struct dwc3_event_devt *event)
2749{
2750 switch (event->type) {
2751 case DWC3_DEVICE_EVENT_DISCONNECT:
2752 dwc3_gadget_disconnect_interrupt(dwc);
2753 break;
2754 case DWC3_DEVICE_EVENT_RESET:
2755 dwc3_gadget_reset_interrupt(dwc);
2756 break;
2757 case DWC3_DEVICE_EVENT_CONNECT_DONE:
2758 dwc3_gadget_conndone_interrupt(dwc);
2759 break;
2760 case DWC3_DEVICE_EVENT_WAKEUP:
2761 dwc3_gadget_wakeup_interrupt(dwc);
2762 break;
Felipe Balbie1dadd32014-02-25 14:47:54 -06002763 case DWC3_DEVICE_EVENT_HIBER_REQ:
2764 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2765 "unexpected hibernation event\n"))
2766 break;
2767
2768 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2769 break;
Felipe Balbi72246da2011-08-19 18:10:58 +03002770 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2771 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2772 break;
2773 case DWC3_DEVICE_EVENT_EOPF:
Baolin Wang72704f82016-05-16 16:43:53 +08002774 /* It changed to be suspend event for version 2.30a and above */
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02002775 if (dwc->revision >= DWC3_REVISION_230A) {
Baolin Wang72704f82016-05-16 16:43:53 +08002776 /*
2777 * Ignore suspend event until the gadget enters into
2778 * USB_STATE_CONFIGURED state.
2779 */
2780 if (dwc->gadget.state >= USB_STATE_CONFIGURED)
2781 dwc3_gadget_suspend_interrupt(dwc,
2782 event->event_info);
2783 }
Felipe Balbi72246da2011-08-19 18:10:58 +03002784 break;
2785 case DWC3_DEVICE_EVENT_SOF:
Felipe Balbi72246da2011-08-19 18:10:58 +03002786 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
Felipe Balbi72246da2011-08-19 18:10:58 +03002787 case DWC3_DEVICE_EVENT_CMD_CMPL:
Felipe Balbi72246da2011-08-19 18:10:58 +03002788 case DWC3_DEVICE_EVENT_OVERFLOW:
Felipe Balbi72246da2011-08-19 18:10:58 +03002789 break;
2790 default:
Felipe Balbie9f2aa82015-01-27 13:49:28 -06002791 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
Felipe Balbi72246da2011-08-19 18:10:58 +03002792 }
2793}
2794
2795static void dwc3_process_event_entry(struct dwc3 *dwc,
2796 const union dwc3_event *event)
2797{
Felipe Balbi43c96be2016-09-26 13:23:34 +03002798 trace_dwc3_event(event->raw, dwc);
Felipe Balbi2c4cbe6e52014-04-30 17:45:10 -05002799
Felipe Balbi72246da2011-08-19 18:10:58 +03002800 /* Endpoint IRQ, handle it and return early */
2801 if (event->type.is_devspec == 0) {
2802 /* depevt */
2803 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2804 }
2805
2806 switch (event->type.type) {
2807 case DWC3_EVENT_TYPE_DEV:
2808 dwc3_gadget_interrupt(dwc, &event->devt);
2809 break;
2810 /* REVISIT what to do with Carkit and I2C events ? */
2811 default:
2812 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2813 }
2814}
2815
Felipe Balbidea520a2016-03-30 09:39:34 +03002816static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
Felipe Balbif42f2442013-06-12 21:25:08 +03002817{
Felipe Balbidea520a2016-03-30 09:39:34 +03002818 struct dwc3 *dwc = evt->dwc;
Felipe Balbif42f2442013-06-12 21:25:08 +03002819 irqreturn_t ret = IRQ_NONE;
2820 int left;
2821 u32 reg;
2822
Felipe Balbif42f2442013-06-12 21:25:08 +03002823 left = evt->count;
2824
2825 if (!(evt->flags & DWC3_EVENT_PENDING))
2826 return IRQ_NONE;
2827
2828 while (left > 0) {
2829 union dwc3_event event;
2830
2831 event.raw = *(u32 *) (evt->buf + evt->lpos);
2832
2833 dwc3_process_event_entry(dwc, &event);
2834
2835 /*
2836 * FIXME we wrap around correctly to the next entry as
2837 * almost all entries are 4 bytes in size. There is one
2838 * entry which has 12 bytes which is a regular entry
2839 * followed by 8 bytes data. ATM I don't know how
2840 * things are organized if we get next to the a
2841 * boundary so I worry about that once we try to handle
2842 * that.
2843 */
2844 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2845 left -= 4;
2846
Felipe Balbi660e9bd2016-03-30 09:26:24 +03002847 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 4);
Felipe Balbif42f2442013-06-12 21:25:08 +03002848 }
2849
2850 evt->count = 0;
2851 evt->flags &= ~DWC3_EVENT_PENDING;
2852 ret = IRQ_HANDLED;
2853
2854 /* Unmask interrupt */
Felipe Balbi660e9bd2016-03-30 09:26:24 +03002855 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
Felipe Balbif42f2442013-06-12 21:25:08 +03002856 reg &= ~DWC3_GEVNTSIZ_INTMASK;
Felipe Balbi660e9bd2016-03-30 09:26:24 +03002857 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
Felipe Balbif42f2442013-06-12 21:25:08 +03002858
2859 return ret;
2860}
2861
Felipe Balbidea520a2016-03-30 09:39:34 +03002862static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
Felipe Balbib15a7622011-06-30 16:57:15 +03002863{
Felipe Balbidea520a2016-03-30 09:39:34 +03002864 struct dwc3_event_buffer *evt = _evt;
2865 struct dwc3 *dwc = evt->dwc;
Felipe Balbie5f68b42015-10-12 13:25:44 -05002866 unsigned long flags;
Felipe Balbib15a7622011-06-30 16:57:15 +03002867 irqreturn_t ret = IRQ_NONE;
Felipe Balbib15a7622011-06-30 16:57:15 +03002868
Felipe Balbie5f68b42015-10-12 13:25:44 -05002869 spin_lock_irqsave(&dwc->lock, flags);
Felipe Balbidea520a2016-03-30 09:39:34 +03002870 ret = dwc3_process_event_buf(evt);
Felipe Balbie5f68b42015-10-12 13:25:44 -05002871 spin_unlock_irqrestore(&dwc->lock, flags);
Felipe Balbib15a7622011-06-30 16:57:15 +03002872
2873 return ret;
2874}
2875
Felipe Balbidea520a2016-03-30 09:39:34 +03002876static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
Felipe Balbi72246da2011-08-19 18:10:58 +03002877{
Felipe Balbidea520a2016-03-30 09:39:34 +03002878 struct dwc3 *dwc = evt->dwc;
Felipe Balbi72246da2011-08-19 18:10:58 +03002879 u32 count;
Felipe Balbie8adfc32013-06-12 21:11:14 +03002880 u32 reg;
Felipe Balbi72246da2011-08-19 18:10:58 +03002881
Felipe Balbifc8bb912016-05-16 13:14:48 +03002882 if (pm_runtime_suspended(dwc->dev)) {
2883 pm_runtime_get(dwc->dev);
2884 disable_irq_nosync(dwc->irq_gadget);
2885 dwc->pending_events = true;
2886 return IRQ_HANDLED;
2887 }
2888
Felipe Balbi660e9bd2016-03-30 09:26:24 +03002889 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
Felipe Balbi72246da2011-08-19 18:10:58 +03002890 count &= DWC3_GEVNTCOUNT_MASK;
2891 if (!count)
2892 return IRQ_NONE;
2893
Felipe Balbib15a7622011-06-30 16:57:15 +03002894 evt->count = count;
2895 evt->flags |= DWC3_EVENT_PENDING;
Felipe Balbi72246da2011-08-19 18:10:58 +03002896
Felipe Balbie8adfc32013-06-12 21:11:14 +03002897 /* Mask interrupt */
Felipe Balbi660e9bd2016-03-30 09:26:24 +03002898 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
Felipe Balbie8adfc32013-06-12 21:11:14 +03002899 reg |= DWC3_GEVNTSIZ_INTMASK;
Felipe Balbi660e9bd2016-03-30 09:26:24 +03002900 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
Felipe Balbie8adfc32013-06-12 21:11:14 +03002901
Felipe Balbib15a7622011-06-30 16:57:15 +03002902 return IRQ_WAKE_THREAD;
Felipe Balbi72246da2011-08-19 18:10:58 +03002903}
2904
Felipe Balbidea520a2016-03-30 09:39:34 +03002905static irqreturn_t dwc3_interrupt(int irq, void *_evt)
Felipe Balbi72246da2011-08-19 18:10:58 +03002906{
Felipe Balbidea520a2016-03-30 09:39:34 +03002907 struct dwc3_event_buffer *evt = _evt;
Felipe Balbi72246da2011-08-19 18:10:58 +03002908
Felipe Balbidea520a2016-03-30 09:39:34 +03002909 return dwc3_check_event_buf(evt);
Felipe Balbi72246da2011-08-19 18:10:58 +03002910}
2911
Felipe Balbi6db38122016-10-03 11:27:01 +03002912static int dwc3_gadget_get_irq(struct dwc3 *dwc)
2913{
2914 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
2915 int irq;
2916
2917 irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
2918 if (irq > 0)
2919 goto out;
2920
2921 if (irq == -EPROBE_DEFER)
2922 goto out;
2923
2924 irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
2925 if (irq > 0)
2926 goto out;
2927
2928 if (irq == -EPROBE_DEFER)
2929 goto out;
2930
2931 irq = platform_get_irq(dwc3_pdev, 0);
2932 if (irq > 0)
2933 goto out;
2934
2935 if (irq != -EPROBE_DEFER)
2936 dev_err(dwc->dev, "missing peripheral IRQ\n");
2937
2938 if (!irq)
2939 irq = -EINVAL;
2940
2941out:
2942 return irq;
2943}
2944
Felipe Balbi72246da2011-08-19 18:10:58 +03002945/**
2946 * dwc3_gadget_init - Initializes gadget related registers
Paul Zimmerman1d046792012-02-15 18:56:56 -08002947 * @dwc: pointer to our controller context structure
Felipe Balbi72246da2011-08-19 18:10:58 +03002948 *
2949 * Returns 0 on success otherwise negative errno.
2950 */
Bill Pemberton41ac7b32012-11-19 13:21:48 -05002951int dwc3_gadget_init(struct dwc3 *dwc)
Felipe Balbi72246da2011-08-19 18:10:58 +03002952{
Felipe Balbi6db38122016-10-03 11:27:01 +03002953 int ret;
2954 int irq;
Roger Quadros9522def2016-06-10 14:48:38 +03002955
Felipe Balbi6db38122016-10-03 11:27:01 +03002956 irq = dwc3_gadget_get_irq(dwc);
2957 if (irq < 0) {
2958 ret = irq;
2959 goto err0;
Roger Quadros9522def2016-06-10 14:48:38 +03002960 }
2961
2962 dwc->irq_gadget = irq;
Felipe Balbi72246da2011-08-19 18:10:58 +03002963
2964 dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2965 &dwc->ctrl_req_addr, GFP_KERNEL);
2966 if (!dwc->ctrl_req) {
2967 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2968 ret = -ENOMEM;
2969 goto err0;
2970 }
2971
Kishon Vijay Abraham I2abd9d52015-07-27 12:25:31 +05302972 dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
Felipe Balbi72246da2011-08-19 18:10:58 +03002973 &dwc->ep0_trb_addr, GFP_KERNEL);
2974 if (!dwc->ep0_trb) {
2975 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2976 ret = -ENOMEM;
2977 goto err1;
2978 }
2979
Felipe Balbi3ef35fa2012-05-04 12:58:14 +03002980 dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
Felipe Balbi72246da2011-08-19 18:10:58 +03002981 if (!dwc->setup_buf) {
Felipe Balbi72246da2011-08-19 18:10:58 +03002982 ret = -ENOMEM;
2983 goto err2;
2984 }
2985
Felipe Balbi5812b1c2011-08-27 22:07:53 +03002986 dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
Felipe Balbi3ef35fa2012-05-04 12:58:14 +03002987 DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2988 GFP_KERNEL);
Felipe Balbi5812b1c2011-08-27 22:07:53 +03002989 if (!dwc->ep0_bounce) {
2990 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2991 ret = -ENOMEM;
2992 goto err3;
2993 }
2994
Felipe Balbi04c03d12015-12-02 10:06:45 -06002995 dwc->zlp_buf = kzalloc(DWC3_ZLP_BUF_SIZE, GFP_KERNEL);
2996 if (!dwc->zlp_buf) {
2997 ret = -ENOMEM;
2998 goto err4;
2999 }
3000
Baolin Wangbb014732016-10-14 17:11:33 +08003001 init_completion(&dwc->ep0_in_setup);
3002
Felipe Balbi72246da2011-08-19 18:10:58 +03003003 dwc->gadget.ops = &dwc3_gadget_ops;
Felipe Balbi72246da2011-08-19 18:10:58 +03003004 dwc->gadget.speed = USB_SPEED_UNKNOWN;
Felipe Balbieeb720f2011-11-28 12:46:59 +02003005 dwc->gadget.sg_supported = true;
Felipe Balbi72246da2011-08-19 18:10:58 +03003006 dwc->gadget.name = "dwc3-gadget";
Jianqiang Tang6a4290c2016-01-20 14:09:39 +08003007 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG;
Felipe Balbi72246da2011-08-19 18:10:58 +03003008
3009 /*
Ben McCauleyb9e51b22015-11-16 10:47:24 -06003010 * FIXME We might be setting max_speed to <SUPER, however versions
3011 * <2.20a of dwc3 have an issue with metastability (documented
3012 * elsewhere in this driver) which tells us we can't set max speed to
3013 * anything lower than SUPER.
3014 *
3015 * Because gadget.max_speed is only used by composite.c and function
3016 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3017 * to happen so we avoid sending SuperSpeed Capability descriptor
3018 * together with our BOS descriptor as that could confuse host into
3019 * thinking we can handle super speed.
3020 *
3021 * Note that, in fact, we won't even support GetBOS requests when speed
3022 * is less than super speed because we don't have means, yet, to tell
3023 * composite.c that we are USB 2.0 + LPM ECN.
3024 */
3025 if (dwc->revision < DWC3_REVISION_220A)
Felipe Balbi5eb30ce2016-11-03 14:07:51 +02003026 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
Ben McCauleyb9e51b22015-11-16 10:47:24 -06003027 dwc->revision);
3028
3029 dwc->gadget.max_speed = dwc->maximum_speed;
3030
3031 /*
David Cohena4b9d942013-12-09 15:55:38 -08003032 * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
3033 * on ep out.
3034 */
3035 dwc->gadget.quirk_ep_out_aligned_size = true;
3036
3037 /*
Felipe Balbi72246da2011-08-19 18:10:58 +03003038 * REVISIT: Here we should clear all pending IRQs to be
3039 * sure we're starting from a well known location.
3040 */
3041
3042 ret = dwc3_gadget_init_endpoints(dwc);
3043 if (ret)
Felipe Balbi04c03d12015-12-02 10:06:45 -06003044 goto err5;
Felipe Balbi72246da2011-08-19 18:10:58 +03003045
Felipe Balbi72246da2011-08-19 18:10:58 +03003046 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3047 if (ret) {
3048 dev_err(dwc->dev, "failed to register udc\n");
Felipe Balbi04c03d12015-12-02 10:06:45 -06003049 goto err5;
Felipe Balbi72246da2011-08-19 18:10:58 +03003050 }
3051
3052 return 0;
3053
Felipe Balbi04c03d12015-12-02 10:06:45 -06003054err5:
3055 kfree(dwc->zlp_buf);
3056
Felipe Balbi5812b1c2011-08-27 22:07:53 +03003057err4:
David Cohene1f80462013-09-11 17:42:47 -07003058 dwc3_gadget_free_endpoints(dwc);
Felipe Balbi3ef35fa2012-05-04 12:58:14 +03003059 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
3060 dwc->ep0_bounce, dwc->ep0_bounce_addr);
Felipe Balbi5812b1c2011-08-27 22:07:53 +03003061
Felipe Balbi72246da2011-08-19 18:10:58 +03003062err3:
Felipe Balbi0fc9a1b2011-12-19 11:32:34 +02003063 kfree(dwc->setup_buf);
Felipe Balbi72246da2011-08-19 18:10:58 +03003064
3065err2:
Christophe JAILLET51fbc7c2016-10-07 22:12:39 +02003066 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
Felipe Balbi72246da2011-08-19 18:10:58 +03003067 dwc->ep0_trb, dwc->ep0_trb_addr);
3068
3069err1:
3070 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
3071 dwc->ctrl_req, dwc->ctrl_req_addr);
3072
3073err0:
3074 return ret;
3075}
3076
Felipe Balbi7415f172012-04-30 14:56:33 +03003077/* -------------------------------------------------------------------------- */
3078
Felipe Balbi72246da2011-08-19 18:10:58 +03003079void dwc3_gadget_exit(struct dwc3 *dwc)
3080{
Felipe Balbi72246da2011-08-19 18:10:58 +03003081 usb_del_gadget_udc(&dwc->gadget);
Felipe Balbi72246da2011-08-19 18:10:58 +03003082
Felipe Balbi72246da2011-08-19 18:10:58 +03003083 dwc3_gadget_free_endpoints(dwc);
3084
Felipe Balbi3ef35fa2012-05-04 12:58:14 +03003085 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
3086 dwc->ep0_bounce, dwc->ep0_bounce_addr);
Felipe Balbi5812b1c2011-08-27 22:07:53 +03003087
Felipe Balbi0fc9a1b2011-12-19 11:32:34 +02003088 kfree(dwc->setup_buf);
Felipe Balbi04c03d12015-12-02 10:06:45 -06003089 kfree(dwc->zlp_buf);
Felipe Balbi72246da2011-08-19 18:10:58 +03003090
Christophe JAILLET51fbc7c2016-10-07 22:12:39 +02003091 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
Felipe Balbi72246da2011-08-19 18:10:58 +03003092 dwc->ep0_trb, dwc->ep0_trb_addr);
3093
3094 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
3095 dwc->ctrl_req, dwc->ctrl_req_addr);
Felipe Balbi72246da2011-08-19 18:10:58 +03003096}
Felipe Balbi7415f172012-04-30 14:56:33 +03003097
Felipe Balbi0b0231a2014-10-07 10:19:23 -05003098int dwc3_gadget_suspend(struct dwc3 *dwc)
Felipe Balbi7415f172012-04-30 14:56:33 +03003099{
Felipe Balbi9f8a67b2016-05-04 15:50:27 +03003100 int ret;
3101
Roger Quadros9772b472016-04-12 11:33:29 +03003102 if (!dwc->gadget_driver)
3103 return 0;
3104
Felipe Balbi9f8a67b2016-05-04 15:50:27 +03003105 ret = dwc3_gadget_run_stop(dwc, false, false);
3106 if (ret < 0)
3107 return ret;
Felipe Balbi7415f172012-04-30 14:56:33 +03003108
Felipe Balbi9f8a67b2016-05-04 15:50:27 +03003109 dwc3_disconnect_gadget(dwc);
3110 __dwc3_gadget_stop(dwc);
Felipe Balbi7415f172012-04-30 14:56:33 +03003111
3112 return 0;
3113}
3114
3115int dwc3_gadget_resume(struct dwc3 *dwc)
3116{
Felipe Balbi7415f172012-04-30 14:56:33 +03003117 int ret;
3118
Roger Quadros9772b472016-04-12 11:33:29 +03003119 if (!dwc->gadget_driver)
3120 return 0;
3121
Felipe Balbi9f8a67b2016-05-04 15:50:27 +03003122 ret = __dwc3_gadget_start(dwc);
3123 if (ret < 0)
Felipe Balbi7415f172012-04-30 14:56:33 +03003124 goto err0;
3125
Felipe Balbi9f8a67b2016-05-04 15:50:27 +03003126 ret = dwc3_gadget_run_stop(dwc, true, false);
3127 if (ret < 0)
Felipe Balbi7415f172012-04-30 14:56:33 +03003128 goto err1;
3129
Felipe Balbi7415f172012-04-30 14:56:33 +03003130 return 0;
3131
3132err1:
Felipe Balbi9f8a67b2016-05-04 15:50:27 +03003133 __dwc3_gadget_stop(dwc);
Felipe Balbi7415f172012-04-30 14:56:33 +03003134
3135err0:
3136 return ret;
3137}
Felipe Balbifc8bb912016-05-16 13:14:48 +03003138
3139void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3140{
3141 if (dwc->pending_events) {
3142 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3143 dwc->pending_events = false;
3144 enable_irq(dwc->irq_gadget);
3145 }
3146}