blob: 1dfe54f147370711da9884066164cc8913a4c5ee [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (c) 2001-2004 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
David Brownell53bd6a62006-08-30 14:50:06 -07004 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20/* this file is part of ehci-hcd.c */
21
22/*-------------------------------------------------------------------------*/
23
24/*
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
27 *
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
31 *
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
35 */
36
Geyslan G. Bemee906472016-01-25 22:45:14 -030037static int ehci_get_frame(struct usb_hcd *hcd);
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
Linus Torvalds1da177e2005-04-16 15:20:36 -070039/*
40 * periodic_next_shadow - return "next" pointer on shadow list
41 * @periodic: host pointer to qh/itd/sitd
42 * @tag: hardware tag for type of this record
43 */
44static union ehci_shadow *
Stefan Roese6dbd6822007-05-01 09:29:37 -070045periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
46 __hc32 tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -070047{
Stefan Roese6dbd6822007-05-01 09:29:37 -070048 switch (hc32_to_cpu(ehci, tag)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 case Q_TYPE_QH:
50 return &periodic->qh->qh_next;
51 case Q_TYPE_FSTN:
52 return &periodic->fstn->fstn_next;
53 case Q_TYPE_ITD:
54 return &periodic->itd->itd_next;
Geyslan G. Bem1ec27802016-01-25 22:45:16 -030055 /* case Q_TYPE_SITD: */
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 default:
57 return &periodic->sitd->sitd_next;
58 }
59}
60
Alek Du3807e262009-07-14 07:23:29 +080061static __hc32 *
62shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
63 __hc32 tag)
64{
65 switch (hc32_to_cpu(ehci, tag)) {
66 /* our ehci_shadow.qh is actually software part */
67 case Q_TYPE_QH:
68 return &periodic->qh->hw->hw_next;
69 /* others are hw parts */
70 default:
71 return periodic->hw_next;
72 }
73}
74
Linus Torvalds1da177e2005-04-16 15:20:36 -070075/* caller must hold ehci->lock */
Geyslan G. Bemee906472016-01-25 22:45:14 -030076static void periodic_unlink(struct ehci_hcd *ehci, unsigned frame, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -070077{
Stefan Roese6dbd6822007-05-01 09:29:37 -070078 union ehci_shadow *prev_p = &ehci->pshadow[frame];
79 __hc32 *hw_p = &ehci->periodic[frame];
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 union ehci_shadow here = *prev_p;
81
82 /* find predecessor of "ptr"; hw and shadow lists are in sync */
83 while (here.ptr && here.ptr != ptr) {
Stefan Roese6dbd6822007-05-01 09:29:37 -070084 prev_p = periodic_next_shadow(ehci, prev_p,
85 Q_NEXT_TYPE(ehci, *hw_p));
Alek Du3807e262009-07-14 07:23:29 +080086 hw_p = shadow_next_periodic(ehci, &here,
87 Q_NEXT_TYPE(ehci, *hw_p));
Linus Torvalds1da177e2005-04-16 15:20:36 -070088 here = *prev_p;
89 }
90 /* an interrupt entry (at list end) could have been shared */
91 if (!here.ptr)
92 return;
93
94 /* update shadow and hardware lists ... the old "next" pointers
95 * from ptr may still be in use, the caller updates them.
96 */
Stefan Roese6dbd6822007-05-01 09:29:37 -070097 *prev_p = *periodic_next_shadow(ehci, &here,
98 Q_NEXT_TYPE(ehci, *hw_p));
Andiry Xu3d091a62010-11-08 17:58:35 +080099
100 if (!ehci->use_dummy_qh ||
101 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p))
102 != EHCI_LIST_END(ehci))
103 *hw_p = *shadow_next_periodic(ehci, &here,
104 Q_NEXT_TYPE(ehci, *hw_p));
105 else
Alan Stern4a71f242013-10-18 11:15:14 -0400106 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107}
108
Alan Sternb35c5002013-10-11 22:16:21 -0400109/*-------------------------------------------------------------------------*/
110
111/* Bandwidth and TT management */
112
113/* Find the TT data structure for this device; create it if necessary */
114static struct ehci_tt *find_tt(struct usb_device *udev)
115{
116 struct usb_tt *utt = udev->tt;
117 struct ehci_tt *tt, **tt_index, **ptt;
118 unsigned port;
119 bool allocated_index = false;
120
121 if (!utt)
122 return NULL; /* Not below a TT */
123
124 /*
125 * Find/create our data structure.
126 * For hubs with a single TT, we get it directly.
127 * For hubs with multiple TTs, there's an extra level of pointers.
128 */
129 tt_index = NULL;
130 if (utt->multi) {
131 tt_index = utt->hcpriv;
132 if (!tt_index) { /* Create the index array */
133 tt_index = kzalloc(utt->hub->maxchild *
134 sizeof(*tt_index), GFP_ATOMIC);
135 if (!tt_index)
136 return ERR_PTR(-ENOMEM);
137 utt->hcpriv = tt_index;
138 allocated_index = true;
139 }
140 port = udev->ttport - 1;
141 ptt = &tt_index[port];
142 } else {
143 port = 0;
144 ptt = (struct ehci_tt **) &utt->hcpriv;
145 }
146
147 tt = *ptt;
148 if (!tt) { /* Create the ehci_tt */
149 struct ehci_hcd *ehci =
150 hcd_to_ehci(bus_to_hcd(udev->bus));
151
152 tt = kzalloc(sizeof(*tt), GFP_ATOMIC);
153 if (!tt) {
154 if (allocated_index) {
155 utt->hcpriv = NULL;
156 kfree(tt_index);
157 }
158 return ERR_PTR(-ENOMEM);
159 }
160 list_add_tail(&tt->tt_list, &ehci->tt_list);
161 INIT_LIST_HEAD(&tt->ps_list);
162 tt->usb_tt = utt;
163 tt->tt_port = port;
164 *ptt = tt;
165 }
166
167 return tt;
168}
169
170/* Release the TT above udev, if it's not in use */
171static void drop_tt(struct usb_device *udev)
172{
173 struct usb_tt *utt = udev->tt;
174 struct ehci_tt *tt, **tt_index, **ptt;
175 int cnt, i;
176
177 if (!utt || !utt->hcpriv)
178 return; /* Not below a TT, or never allocated */
179
180 cnt = 0;
181 if (utt->multi) {
182 tt_index = utt->hcpriv;
183 ptt = &tt_index[udev->ttport - 1];
184
185 /* How many entries are left in tt_index? */
186 for (i = 0; i < utt->hub->maxchild; ++i)
187 cnt += !!tt_index[i];
188 } else {
189 tt_index = NULL;
190 ptt = (struct ehci_tt **) &utt->hcpriv;
191 }
192
193 tt = *ptt;
194 if (!tt || !list_empty(&tt->ps_list))
195 return; /* never allocated, or still in use */
196
197 list_del(&tt->tt_list);
198 *ptt = NULL;
199 kfree(tt);
200 if (cnt == 1) {
201 utt->hcpriv = NULL;
202 kfree(tt_index);
203 }
204}
205
Alan Sternd0ce5c62013-10-11 11:29:13 -0400206static void bandwidth_dbg(struct ehci_hcd *ehci, int sign, char *type,
207 struct ehci_per_sched *ps)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208{
Alan Sternd0ce5c62013-10-11 11:29:13 -0400209 dev_dbg(&ps->udev->dev,
210 "ep %02x: %s %s @ %u+%u (%u.%u+%u) [%u/%u us] mask %04x\n",
211 ps->ep->desc.bEndpointAddress,
212 (sign >= 0 ? "reserve" : "release"), type,
213 (ps->bw_phase << 3) + ps->phase_uf, ps->bw_uperiod,
214 ps->phase, ps->phase_uf, ps->period,
215 ps->usecs, ps->c_usecs, ps->cs_mask);
216}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
Alan Sternd0ce5c62013-10-11 11:29:13 -0400218static void reserve_release_intr_bandwidth(struct ehci_hcd *ehci,
219 struct ehci_qh *qh, int sign)
220{
221 unsigned start_uf;
222 unsigned i, j, m;
223 int usecs = qh->ps.usecs;
224 int c_usecs = qh->ps.c_usecs;
Alan Sternb35c5002013-10-11 22:16:21 -0400225 int tt_usecs = qh->ps.tt_usecs;
226 struct ehci_tt *tt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227
Alan Sternd0ce5c62013-10-11 11:29:13 -0400228 if (qh->ps.phase == NO_FRAME) /* Bandwidth wasn't reserved */
229 return;
230 start_uf = qh->ps.bw_phase << 3;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231
Alan Sternd0ce5c62013-10-11 11:29:13 -0400232 bandwidth_dbg(ehci, sign, "intr", &qh->ps);
233
234 if (sign < 0) { /* Release bandwidth */
235 usecs = -usecs;
236 c_usecs = -c_usecs;
Alan Sternb35c5002013-10-11 22:16:21 -0400237 tt_usecs = -tt_usecs;
Alan Sternd0ce5c62013-10-11 11:29:13 -0400238 }
239
240 /* Entire transaction (high speed) or start-split (full/low speed) */
241 for (i = start_uf + qh->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
242 i += qh->ps.bw_uperiod)
243 ehci->bandwidth[i] += usecs;
244
245 /* Complete-split (full/low speed) */
246 if (qh->ps.c_usecs) {
247 /* NOTE: adjustments needed for FSTN */
248 for (i = start_uf; i < EHCI_BANDWIDTH_SIZE;
249 i += qh->ps.bw_uperiod) {
250 for ((j = 2, m = 1 << (j+8)); j < 8; (++j, m <<= 1)) {
251 if (qh->ps.cs_mask & m)
252 ehci->bandwidth[i+j] += c_usecs;
253 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 }
255 }
Alan Sternb35c5002013-10-11 22:16:21 -0400256
257 /* FS/LS bus bandwidth */
258 if (tt_usecs) {
259 tt = find_tt(qh->ps.udev);
260 if (sign > 0)
261 list_add_tail(&qh->ps.ps_list, &tt->ps_list);
262 else
263 list_del(&qh->ps.ps_list);
264
265 for (i = start_uf >> 3; i < EHCI_BANDWIDTH_FRAMES;
266 i += qh->ps.bw_period)
267 tt->bandwidth[i] += tt_usecs;
268 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269}
270
271/*-------------------------------------------------------------------------*/
272
Alan Sternb35c5002013-10-11 22:16:21 -0400273static void compute_tt_budget(u8 budget_table[EHCI_BANDWIDTH_SIZE],
274 struct ehci_tt *tt)
275{
276 struct ehci_per_sched *ps;
277 unsigned uframe, uf, x;
278 u8 *budget_line;
279
280 if (!tt)
281 return;
282 memset(budget_table, 0, EHCI_BANDWIDTH_SIZE);
283
284 /* Add up the contributions from all the endpoints using this TT */
285 list_for_each_entry(ps, &tt->ps_list, ps_list) {
286 for (uframe = ps->bw_phase << 3; uframe < EHCI_BANDWIDTH_SIZE;
287 uframe += ps->bw_uperiod) {
288 budget_line = &budget_table[uframe];
289 x = ps->tt_usecs;
290
291 /* propagate the time forward */
292 for (uf = ps->phase_uf; uf < 8; ++uf) {
293 x += budget_line[uf];
294
295 /* Each microframe lasts 125 us */
296 if (x <= 125) {
297 budget_line[uf] = x;
298 break;
Alan Sternb35c5002013-10-11 22:16:21 -0400299 }
Geyslan G. Bem2fee2fe2016-01-25 22:45:15 -0300300 budget_line[uf] = 125;
301 x -= 125;
Alan Sternb35c5002013-10-11 22:16:21 -0400302 }
303 }
304 }
305}
306
307static int __maybe_unused same_tt(struct usb_device *dev1,
308 struct usb_device *dev2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309{
310 if (!dev1->tt || !dev2->tt)
311 return 0;
312 if (dev1->tt != dev2->tt)
313 return 0;
314 if (dev1->tt->multi)
315 return dev1->ttport == dev2->ttport;
316 else
317 return 1;
318}
319
Dan Streetmanba47f662006-05-24 09:39:16 -0700320#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
321
322/* Which uframe does the low/fullspeed transfer start in?
323 *
324 * The parameter is the mask of ssplits in "H-frame" terms
325 * and this returns the transfer start uframe in "B-frame" terms,
326 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
327 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag
328 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7.
329 */
Stefan Roese6dbd6822007-05-01 09:29:37 -0700330static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
Dan Streetmanba47f662006-05-24 09:39:16 -0700331{
Geyslan G. Bem25d10862016-01-25 22:45:11 -0300332 unsigned char smask = hc32_to_cpu(ehci, mask) & QH_SMASK;
Geyslan G. Bem6d0febc2016-01-25 22:45:17 -0300333
Dan Streetmanba47f662006-05-24 09:39:16 -0700334 if (!smask) {
335 ehci_err(ehci, "invalid empty smask!\n");
336 /* uframe 7 can't have bw so this will indicate failure */
337 return 7;
338 }
339 return ffs(smask) - 1;
340}
341
342static const unsigned char
Alan Sternfdc034382013-05-28 14:03:10 -0400343max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
Dan Streetmanba47f662006-05-24 09:39:16 -0700344
345/* carryover low/fullspeed bandwidth that crosses uframe boundries */
346static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
347{
348 int i;
Geyslan G. Bem6d0febc2016-01-25 22:45:17 -0300349
Geyslan G. Bem189b8ff2016-01-25 22:45:13 -0300350 for (i = 0; i < 7; i++) {
Dan Streetmanba47f662006-05-24 09:39:16 -0700351 if (max_tt_usecs[i] < tt_usecs[i]) {
352 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
353 tt_usecs[i] = max_tt_usecs[i];
354 }
355 }
356}
357
Dan Streetmanba47f662006-05-24 09:39:16 -0700358/*
359 * Return true if the device's tt's downstream bus is available for a
360 * periodic transfer of the specified length (usecs), starting at the
361 * specified frame/uframe. Note that (as summarized in section 11.19
362 * of the usb 2.0 spec) TTs can buffer multiple transactions for each
363 * uframe.
364 *
365 * The uframe parameter is when the fullspeed/lowspeed transfer
366 * should be executed in "B-frame" terms, which is the same as the
367 * highspeed ssplit's uframe (which is in "H-frame" terms). For example
368 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
369 * See the EHCI spec sec 4.5 and fig 4.7.
370 *
371 * This checks if the full/lowspeed bus, at the specified starting uframe,
372 * has the specified bandwidth available, according to rules listed
373 * in USB 2.0 spec section 11.18.1 fig 11-60.
374 *
375 * This does not check if the transfer would exceed the max ssplit
376 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
377 * since proper scheduling limits ssplits to less than 16 per uframe.
378 */
Geyslan G. Bemee906472016-01-25 22:45:14 -0300379static int tt_available(
Dan Streetmanba47f662006-05-24 09:39:16 -0700380 struct ehci_hcd *ehci,
Alan Sternb35c5002013-10-11 22:16:21 -0400381 struct ehci_per_sched *ps,
382 struct ehci_tt *tt,
Dan Streetmanba47f662006-05-24 09:39:16 -0700383 unsigned frame,
Alan Sternb35c5002013-10-11 22:16:21 -0400384 unsigned uframe
Dan Streetmanba47f662006-05-24 09:39:16 -0700385)
386{
Alan Sternb35c5002013-10-11 22:16:21 -0400387 unsigned period = ps->bw_period;
388 unsigned usecs = ps->tt_usecs;
389
Dan Streetmanba47f662006-05-24 09:39:16 -0700390 if ((period == 0) || (uframe >= 7)) /* error */
391 return 0;
392
Alan Sternb35c5002013-10-11 22:16:21 -0400393 for (frame &= period - 1; frame < EHCI_BANDWIDTH_FRAMES;
394 frame += period) {
395 unsigned i, uf;
396 unsigned short tt_usecs[8];
Dan Streetmanba47f662006-05-24 09:39:16 -0700397
Alan Sternb35c5002013-10-11 22:16:21 -0400398 if (tt->bandwidth[frame] + usecs > 900)
399 return 0;
400
401 uf = frame << 3;
402 for (i = 0; i < 8; (++i, ++uf))
403 tt_usecs[i] = ehci->tt_budget[uf];
Dan Streetmanba47f662006-05-24 09:39:16 -0700404
Xenia Ragiadakoufea26ef2013-08-29 11:45:10 +0300405 if (max_tt_usecs[uframe] <= tt_usecs[uframe])
Dan Streetmanba47f662006-05-24 09:39:16 -0700406 return 0;
Dan Streetmanba47f662006-05-24 09:39:16 -0700407
408 /* special case for isoc transfers larger than 125us:
409 * the first and each subsequent fully used uframe
410 * must be empty, so as to not illegally delay
411 * already scheduled transactions
412 */
Geyslan G. Bem25d10862016-01-25 22:45:11 -0300413 if (usecs > 125) {
Dan Streetmanc065c602009-04-21 13:37:12 -0400414 int ufs = (usecs / 125);
Alan Sternb35c5002013-10-11 22:16:21 -0400415
Dan Streetmanba47f662006-05-24 09:39:16 -0700416 for (i = uframe; i < (uframe + ufs) && i < 8; i++)
Geyslan G. Bem25d10862016-01-25 22:45:11 -0300417 if (tt_usecs[i] > 0)
Dan Streetmanba47f662006-05-24 09:39:16 -0700418 return 0;
Dan Streetmanba47f662006-05-24 09:39:16 -0700419 }
420
421 tt_usecs[uframe] += usecs;
422
423 carryover_tt_bandwidth(tt_usecs);
424
425 /* fail if the carryover pushed bw past the last uframe's limit */
Xenia Ragiadakoufea26ef2013-08-29 11:45:10 +0300426 if (max_tt_usecs[7] < tt_usecs[7])
Dan Streetmanba47f662006-05-24 09:39:16 -0700427 return 0;
Dan Streetmanba47f662006-05-24 09:39:16 -0700428 }
429
430 return 1;
431}
432
433#else
434
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435/* return true iff the device's transaction translator is available
436 * for a periodic transfer starting at the specified frame, using
437 * all the uframes in the mask.
438 */
Geyslan G. Bemee906472016-01-25 22:45:14 -0300439static int tt_no_collision(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 struct ehci_hcd *ehci,
441 unsigned period,
442 struct usb_device *dev,
443 unsigned frame,
444 u32 uf_mask
445)
446{
447 if (period == 0) /* error */
448 return 0;
449
450 /* note bandwidth wastage: split never follows csplit
451 * (different dev or endpoint) until the next uframe.
452 * calling convention doesn't make that distinction.
453 */
454 for (; frame < ehci->periodic_size; frame += period) {
455 union ehci_shadow here;
Stefan Roese6dbd6822007-05-01 09:29:37 -0700456 __hc32 type;
Alek Du3807e262009-07-14 07:23:29 +0800457 struct ehci_qh_hw *hw;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458
Geyslan G. Bemee906472016-01-25 22:45:14 -0300459 here = ehci->pshadow[frame];
460 type = Q_NEXT_TYPE(ehci, ehci->periodic[frame]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 while (here.ptr) {
Stefan Roese6dbd6822007-05-01 09:29:37 -0700462 switch (hc32_to_cpu(ehci, type)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463 case Q_TYPE_ITD:
Stefan Roese6dbd6822007-05-01 09:29:37 -0700464 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 here = here.itd->itd_next;
466 continue;
467 case Q_TYPE_QH:
Alek Du3807e262009-07-14 07:23:29 +0800468 hw = here.qh->hw;
Alan Sternffa02482013-10-11 11:29:03 -0400469 if (same_tt(dev, here.qh->ps.udev)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 u32 mask;
471
Stefan Roese6dbd6822007-05-01 09:29:37 -0700472 mask = hc32_to_cpu(ehci,
Alek Du3807e262009-07-14 07:23:29 +0800473 hw->hw_info2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 /* "knows" no gap is needed */
475 mask |= mask >> 8;
476 if (mask & uf_mask)
477 break;
478 }
Alek Du3807e262009-07-14 07:23:29 +0800479 type = Q_NEXT_TYPE(ehci, hw->hw_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 here = here.qh->qh_next;
481 continue;
482 case Q_TYPE_SITD:
Geyslan G. Bemee906472016-01-25 22:45:14 -0300483 if (same_tt(dev, here.sitd->urb->dev)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 u16 mask;
485
Stefan Roese6dbd6822007-05-01 09:29:37 -0700486 mask = hc32_to_cpu(ehci, here.sitd
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 ->hw_uframe);
488 /* FIXME assumes no gap for IN! */
489 mask |= mask >> 8;
490 if (mask & uf_mask)
491 break;
492 }
Stefan Roese6dbd6822007-05-01 09:29:37 -0700493 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 here = here.sitd->sitd_next;
495 continue;
Geyslan G. Bem1ec27802016-01-25 22:45:16 -0300496 /* case Q_TYPE_FSTN: */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 default:
Geyslan G. Bemee906472016-01-25 22:45:14 -0300498 ehci_dbg(ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 "periodic frame %d bogus type %d\n",
500 frame, type);
501 }
502
503 /* collision or error */
504 return 0;
505 }
506 }
507
508 /* no collision */
509 return 1;
510}
511
Dan Streetmanba47f662006-05-24 09:39:16 -0700512#endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
513
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514/*-------------------------------------------------------------------------*/
515
Alan Sternb015cb72012-07-11 11:22:10 -0400516static void enable_periodic(struct ehci_hcd *ehci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517{
Alan Stern3ca9aeb2012-07-11 11:22:05 -0400518 if (ehci->periodic_count++)
Alan Sternb015cb72012-07-11 11:22:10 -0400519 return;
David Brownell01c17142008-08-26 23:35:04 -0700520
Alan Stern3ca9aeb2012-07-11 11:22:05 -0400521 /* Stop waiting to turn off the periodic schedule */
522 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_PERIODIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
Alan Stern3ca9aeb2012-07-11 11:22:05 -0400524 /* Don't start the schedule until PSS is 0 */
525 ehci_poll_PSS(ehci);
Alan Stern18aafe62012-07-11 11:23:04 -0400526 turn_on_io_watchdog(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527}
528
Alan Sternb015cb72012-07-11 11:22:10 -0400529static void disable_periodic(struct ehci_hcd *ehci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530{
Alan Stern3ca9aeb2012-07-11 11:22:05 -0400531 if (--ehci->periodic_count)
Alan Sternb015cb72012-07-11 11:22:10 -0400532 return;
David Brownell01c17142008-08-26 23:35:04 -0700533
Alan Stern3ca9aeb2012-07-11 11:22:05 -0400534 /* Don't turn off the schedule until PSS is 1 */
535 ehci_poll_PSS(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536}
537
538/*-------------------------------------------------------------------------*/
539
540/* periodic schedule slots have iso tds (normal or split) first, then a
541 * sparse tree for active interrupt transfers.
542 *
543 * this just links in a qh; caller guarantees uframe masks are set right.
544 * no FSTN support (yet; ehci 0.96+)
545 */
Alan Sternb015cb72012-07-11 11:22:10 -0400546static void qh_link_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547{
548 unsigned i;
Alan Sternffa02482013-10-11 11:29:03 -0400549 unsigned period = qh->ps.period;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550
Alan Sternffa02482013-10-11 11:29:03 -0400551 dev_dbg(&qh->ps.udev->dev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 "link qh%d-%04x/%p start %d [%d/%d us]\n",
Alek Du3807e262009-07-14 07:23:29 +0800553 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
554 & (QH_CMASK | QH_SMASK),
Alan Sternffa02482013-10-11 11:29:03 -0400555 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556
557 /* high bandwidth, or otherwise every microframe */
558 if (period == 0)
559 period = 1;
560
Alan Sternffa02482013-10-11 11:29:03 -0400561 for (i = qh->ps.phase; i < ehci->periodic_size; i += period) {
Stefan Roese6dbd6822007-05-01 09:29:37 -0700562 union ehci_shadow *prev = &ehci->pshadow[i];
563 __hc32 *hw_p = &ehci->periodic[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 union ehci_shadow here = *prev;
Stefan Roese6dbd6822007-05-01 09:29:37 -0700565 __hc32 type = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566
567 /* skip the iso nodes at list head */
568 while (here.ptr) {
Stefan Roese6dbd6822007-05-01 09:29:37 -0700569 type = Q_NEXT_TYPE(ehci, *hw_p);
570 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 break;
Stefan Roese6dbd6822007-05-01 09:29:37 -0700572 prev = periodic_next_shadow(ehci, prev, type);
Alek Du3807e262009-07-14 07:23:29 +0800573 hw_p = shadow_next_periodic(ehci, &here, type);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 here = *prev;
575 }
576
577 /* sorting each branch by period (slow-->fast)
578 * enables sharing interior tree nodes
579 */
580 while (here.ptr && qh != here.qh) {
Alan Sternffa02482013-10-11 11:29:03 -0400581 if (qh->ps.period > here.qh->ps.period)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 break;
583 prev = &here.qh->qh_next;
Alek Du3807e262009-07-14 07:23:29 +0800584 hw_p = &here.qh->hw->hw_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585 here = *prev;
586 }
587 /* link in this qh, unless some earlier pass did that */
588 if (qh != here.qh) {
589 qh->qh_next = here;
590 if (here.qh)
Alek Du3807e262009-07-14 07:23:29 +0800591 qh->hw->hw_next = *hw_p;
Geyslan G. Bemee906472016-01-25 22:45:14 -0300592 wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 prev->qh = qh;
Geyslan G. Bemee906472016-01-25 22:45:14 -0300594 *hw_p = QH_NEXT(ehci, qh->qh_dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 }
596 }
597 qh->qh_state = QH_STATE_LINKED;
Alan Sternef4638f2009-07-31 10:41:40 -0400598 qh->xacterrs = 0;
Alan Sternfcc51842016-01-25 15:42:04 -0500599 qh->unlink_reason = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
Alan Sternffa02482013-10-11 11:29:03 -0400601 /* update per-qh bandwidth for debugfs */
Alan Sternd0ce5c62013-10-11 11:29:13 -0400602 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->ps.bw_period
603 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
Alan Sternffa02482013-10-11 11:29:03 -0400604 : (qh->ps.usecs * 8);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
Alan Stern569b3942012-07-11 11:23:00 -0400606 list_add(&qh->intr_node, &ehci->intr_qh_list);
607
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 /* maybe enable periodic schedule processing */
Alan Stern569b3942012-07-11 11:23:00 -0400609 ++ehci->intr_count;
Alan Sternb015cb72012-07-11 11:22:10 -0400610 enable_periodic(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611}
612
Alan Sternb015cb72012-07-11 11:22:10 -0400613static void qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614{
615 unsigned i;
616 unsigned period;
617
Alan Sterndf202252012-07-11 11:22:26 -0400618 /*
619 * If qh is for a low/full-speed device, simply unlinking it
620 * could interfere with an ongoing split transaction. To unlink
621 * it safely would require setting the QH_INACTIVATE bit and
622 * waiting at least one frame, as described in EHCI 4.12.2.5.
623 *
624 * We won't bother with any of this. Instead, we assume that the
625 * only reason for unlinking an interrupt QH while the current URB
626 * is still active is to dequeue all the URBs (flush the whole
627 * endpoint queue).
628 *
629 * If rebalancing the periodic schedule is ever implemented, this
630 * approach will no longer be valid.
631 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632
633 /* high bandwidth, or otherwise part of every microframe */
Alan Sternffa02482013-10-11 11:29:03 -0400634 period = qh->ps.period ? : 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635
Alan Sternffa02482013-10-11 11:29:03 -0400636 for (i = qh->ps.phase; i < ehci->periodic_size; i += period)
Geyslan G. Bemee906472016-01-25 22:45:14 -0300637 periodic_unlink(ehci, i, qh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
Alan Sternffa02482013-10-11 11:29:03 -0400639 /* update per-qh bandwidth for debugfs */
Alan Sternd0ce5c62013-10-11 11:29:13 -0400640 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->ps.bw_period
641 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period)
Alan Sternffa02482013-10-11 11:29:03 -0400642 : (qh->ps.usecs * 8);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643
Alan Sternffa02482013-10-11 11:29:03 -0400644 dev_dbg(&qh->ps.udev->dev,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
Alan Sternffa02482013-10-11 11:29:03 -0400646 qh->ps.period,
Alek Du3807e262009-07-14 07:23:29 +0800647 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
Alan Sternffa02482013-10-11 11:29:03 -0400648 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649
650 /* qh->qh_next still "live" to HC */
651 qh->qh_state = QH_STATE_UNLINK;
652 qh->qh_next.ptr = NULL;
Alan Stern569b3942012-07-11 11:23:00 -0400653
654 if (ehci->qh_scan_next == qh)
655 ehci->qh_scan_next = list_entry(qh->intr_node.next,
656 struct ehci_qh, intr_node);
657 list_del(&qh->intr_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658}
659
Ming Lei9118f9e2013-07-03 22:53:10 +0800660static void cancel_unlink_wait_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
661{
662 if (qh->qh_state != QH_STATE_LINKED ||
663 list_empty(&qh->unlink_node))
664 return;
665
666 list_del_init(&qh->unlink_node);
667
668 /*
669 * TODO: disable the event of EHCI_HRTIMER_START_UNLINK_INTR for
670 * avoiding unnecessary CPU wakeup
671 */
672}
673
Alan Sterndf202252012-07-11 11:22:26 -0400674static void start_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675{
Alan Stern7bc782d2013-03-22 13:31:11 -0400676 /* If the QH isn't linked then there's nothing we can do. */
677 if (qh->qh_state != QH_STATE_LINKED)
Alan Sterna448c9d2009-08-19 12:22:44 -0400678 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679
Ming Lei9118f9e2013-07-03 22:53:10 +0800680 /* if the qh is waiting for unlink, cancel it now */
681 cancel_unlink_wait_intr(ehci, qh);
682
Geyslan G. Bemee906472016-01-25 22:45:14 -0300683 qh_unlink_periodic(ehci, qh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684
Alan Sterndf202252012-07-11 11:22:26 -0400685 /* Make sure the unlinks are visible before starting the timer */
686 wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687
Alan Sterndf202252012-07-11 11:22:26 -0400688 /*
689 * The EHCI spec doesn't say how long it takes the controller to
690 * stop accessing an unlinked interrupt QH. The timer delay is
691 * 9 uframes; presumably that will be long enough.
692 */
693 qh->unlink_cycle = ehci->intr_unlink_cycle;
694
695 /* New entries go at the end of the intr_unlink list */
Alan Stern6e018752013-03-22 13:31:45 -0400696 list_add_tail(&qh->unlink_node, &ehci->intr_unlink);
Alan Sterndf202252012-07-11 11:22:26 -0400697
698 if (ehci->intr_unlinking)
699 ; /* Avoid recursive calls */
700 else if (ehci->rh_state < EHCI_RH_RUNNING)
701 ehci_handle_intr_unlinks(ehci);
Alan Stern6e018752013-03-22 13:31:45 -0400702 else if (ehci->intr_unlink.next == &qh->unlink_node) {
Alan Sterndf202252012-07-11 11:22:26 -0400703 ehci_enable_event(ehci, EHCI_HRTIMER_UNLINK_INTR, true);
704 ++ehci->intr_unlink_cycle;
705 }
706}
707
Ming Lei9118f9e2013-07-03 22:53:10 +0800708/*
709 * It is common only one intr URB is scheduled on one qh, and
710 * given complete() is run in tasklet context, introduce a bit
711 * delay to avoid unlink qh too early.
712 */
713static void start_unlink_intr_wait(struct ehci_hcd *ehci,
714 struct ehci_qh *qh)
715{
716 qh->unlink_cycle = ehci->intr_unlink_wait_cycle;
717
718 /* New entries go at the end of the intr_unlink_wait list */
719 list_add_tail(&qh->unlink_node, &ehci->intr_unlink_wait);
720
721 if (ehci->rh_state < EHCI_RH_RUNNING)
722 ehci_handle_start_intr_unlinks(ehci);
723 else if (ehci->intr_unlink_wait.next == &qh->unlink_node) {
724 ehci_enable_event(ehci, EHCI_HRTIMER_START_UNLINK_INTR, true);
725 ++ehci->intr_unlink_wait_cycle;
726 }
727}
728
Alan Sterndf202252012-07-11 11:22:26 -0400729static void end_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh)
730{
731 struct ehci_qh_hw *hw = qh->hw;
732 int rc;
733
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 qh->qh_state = QH_STATE_IDLE;
Alek Du3807e262009-07-14 07:23:29 +0800735 hw->hw_next = EHCI_LIST_END(ehci);
Alan Sterna448c9d2009-08-19 12:22:44 -0400736
Alan Stern79bcf7b2013-03-22 13:30:56 -0400737 if (!list_empty(&qh->qtd_list))
738 qh_completions(ehci, qh);
Alan Sterna448c9d2009-08-19 12:22:44 -0400739
740 /* reschedule QH iff another request is queued */
Alan Sterndf202252012-07-11 11:22:26 -0400741 if (!list_empty(&qh->qtd_list) && ehci->rh_state == EHCI_RH_RUNNING) {
Alan Sterna448c9d2009-08-19 12:22:44 -0400742 rc = qh_schedule(ehci, qh);
Alan Stern077f5f12013-05-29 11:33:52 -0400743 if (rc == 0) {
744 qh_refresh(ehci, qh);
745 qh_link_periodic(ehci, qh);
746 }
Alan Sterna448c9d2009-08-19 12:22:44 -0400747
748 /* An error here likely indicates handshake failure
749 * or no space left in the schedule. Neither fault
750 * should happen often ...
751 *
752 * FIXME kill the now-dysfunctional queued urbs
753 */
Alan Stern077f5f12013-05-29 11:33:52 -0400754 else {
Alan Sterna448c9d2009-08-19 12:22:44 -0400755 ehci_err(ehci, "can't reschedule qh %p, err %d\n",
756 qh, rc);
Alan Stern077f5f12013-05-29 11:33:52 -0400757 }
Alan Sterna448c9d2009-08-19 12:22:44 -0400758 }
Alan Stern3ca9aeb2012-07-11 11:22:05 -0400759
760 /* maybe turn off periodic schedule */
Alan Stern569b3942012-07-11 11:23:00 -0400761 --ehci->intr_count;
Alan Stern3ca9aeb2012-07-11 11:22:05 -0400762 disable_periodic(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763}
764
765/*-------------------------------------------------------------------------*/
766
Geyslan G. Bemee906472016-01-25 22:45:14 -0300767static int check_period(
David Brownell53bd6a62006-08-30 14:50:06 -0700768 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 unsigned frame,
770 unsigned uframe,
Alan Sternd0ce5c62013-10-11 11:29:13 -0400771 unsigned uperiod,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 unsigned usecs
773) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 /* complete split running into next frame?
775 * given FSTN support, we could sometimes check...
776 */
777 if (uframe >= 8)
778 return 0;
779
Kirill Smelkovcc62a7e2011-07-03 20:36:57 +0400780 /* convert "usecs we need" to "max already claimed" */
781 usecs = ehci->uframe_periodic_max - usecs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782
Alan Sternd0ce5c62013-10-11 11:29:13 -0400783 for (uframe += frame << 3; uframe < EHCI_BANDWIDTH_SIZE;
784 uframe += uperiod) {
785 if (ehci->bandwidth[uframe] > usecs)
786 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787 }
788
Geyslan G. Bem1ec27802016-01-25 22:45:16 -0300789 /* success! */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 return 1;
791}
792
Geyslan G. Bemee906472016-01-25 22:45:14 -0300793static int check_intr_schedule(
David Brownell53bd6a62006-08-30 14:50:06 -0700794 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 unsigned frame,
796 unsigned uframe,
Alan Sternb35c5002013-10-11 22:16:21 -0400797 struct ehci_qh *qh,
Alan Stern5d8f6812013-10-18 11:13:08 -0400798 unsigned *c_maskp,
Alan Sternb35c5002013-10-11 22:16:21 -0400799 struct ehci_tt *tt
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800)
801{
David Brownell53bd6a62006-08-30 14:50:06 -0700802 int retval = -ENOSPC;
Dan Streetmanba47f662006-05-24 09:39:16 -0700803 u8 mask = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804
Alan Sternffa02482013-10-11 11:29:03 -0400805 if (qh->ps.c_usecs && uframe >= 6) /* FSTN territory? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 goto done;
807
Alan Sternd0ce5c62013-10-11 11:29:13 -0400808 if (!check_period(ehci, frame, uframe, qh->ps.bw_uperiod, qh->ps.usecs))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 goto done;
Alan Sternffa02482013-10-11 11:29:03 -0400810 if (!qh->ps.c_usecs) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811 retval = 0;
812 *c_maskp = 0;
813 goto done;
814 }
815
Dan Streetmanba47f662006-05-24 09:39:16 -0700816#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
Alan Sternb35c5002013-10-11 22:16:21 -0400817 if (tt_available(ehci, &qh->ps, tt, frame, uframe)) {
Dan Streetmanba47f662006-05-24 09:39:16 -0700818 unsigned i;
819
820 /* TODO : this may need FSTN for SSPLIT in uframe 5. */
Alan Sterne24371a2013-10-11 11:28:12 -0400821 for (i = uframe+2; i < 8 && i <= uframe+4; i++)
Alan Sternffa02482013-10-11 11:29:03 -0400822 if (!check_period(ehci, frame, i,
Alan Sternd0ce5c62013-10-11 11:29:13 -0400823 qh->ps.bw_uperiod, qh->ps.c_usecs))
Dan Streetmanba47f662006-05-24 09:39:16 -0700824 goto done;
825 else
826 mask |= 1 << i;
827
828 retval = 0;
829
Alan Sternd0ce5c62013-10-11 11:29:13 -0400830 *c_maskp = mask;
Dan Streetmanba47f662006-05-24 09:39:16 -0700831 }
832#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 /* Make sure this tt's buffer is also available for CSPLITs.
834 * We pessimize a bit; probably the typical full speed case
835 * doesn't need the second CSPLIT.
David Brownell53bd6a62006-08-30 14:50:06 -0700836 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 * NOTE: both SPLIT and CSPLIT could be checked in just
838 * one smart pass...
839 */
840 mask = 0x03 << (uframe + qh->gap_uf);
Alan Sternd0ce5c62013-10-11 11:29:13 -0400841 *c_maskp = mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842
843 mask |= 1 << uframe;
Alan Sternd0ce5c62013-10-11 11:29:13 -0400844 if (tt_no_collision(ehci, qh->ps.bw_period, qh->ps.udev, frame, mask)) {
Alan Sternffa02482013-10-11 11:29:03 -0400845 if (!check_period(ehci, frame, uframe + qh->gap_uf + 1,
Alan Sternd0ce5c62013-10-11 11:29:13 -0400846 qh->ps.bw_uperiod, qh->ps.c_usecs))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 goto done;
Alan Sternffa02482013-10-11 11:29:03 -0400848 if (!check_period(ehci, frame, uframe + qh->gap_uf,
Alan Sternd0ce5c62013-10-11 11:29:13 -0400849 qh->ps.bw_uperiod, qh->ps.c_usecs))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850 goto done;
851 retval = 0;
852 }
Dan Streetmanba47f662006-05-24 09:39:16 -0700853#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854done:
855 return retval;
856}
857
858/* "first fit" scheduling policy used the first time through,
859 * or when the previous schedule slot can't be re-used.
860 */
Stefan Roese6dbd6822007-05-01 09:29:37 -0700861static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862{
Alan Sternb35c5002013-10-11 22:16:21 -0400863 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864 unsigned uframe;
Alan Sternd0ce5c62013-10-11 11:29:13 -0400865 unsigned c_mask;
Alek Du3807e262009-07-14 07:23:29 +0800866 struct ehci_qh_hw *hw = qh->hw;
Alan Sternb35c5002013-10-11 22:16:21 -0400867 struct ehci_tt *tt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
Alek Du3807e262009-07-14 07:23:29 +0800869 hw->hw_next = EHCI_LIST_END(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870
871 /* reuse the previous schedule slots, if we can */
Alan Sternd0ce5c62013-10-11 11:29:13 -0400872 if (qh->ps.phase != NO_FRAME) {
873 ehci_dbg(ehci, "reused qh %p schedule\n", qh);
874 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875 }
876
Alan Sternd0ce5c62013-10-11 11:29:13 -0400877 uframe = 0;
878 c_mask = 0;
Alan Sternb35c5002013-10-11 22:16:21 -0400879 tt = find_tt(qh->ps.udev);
880 if (IS_ERR(tt)) {
881 status = PTR_ERR(tt);
882 goto done;
883 }
884 compute_tt_budget(ehci->tt_budget, tt);
Alan Sternd0ce5c62013-10-11 11:29:13 -0400885
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 /* else scan the schedule to find a group of slots such that all
887 * uframes have enough periodic bandwidth available.
888 */
Alan Sternd0ce5c62013-10-11 11:29:13 -0400889 /* "normal" case, uframing flexible except with splits */
890 if (qh->ps.bw_period) {
891 int i;
892 unsigned frame;
Alan Stern68335e82009-05-22 17:02:33 -0400893
Alan Sternb35c5002013-10-11 22:16:21 -0400894 for (i = qh->ps.bw_period; i > 0; --i) {
Alan Sternd0ce5c62013-10-11 11:29:13 -0400895 frame = ++ehci->random_frame & (qh->ps.bw_period - 1);
896 for (uframe = 0; uframe < 8; uframe++) {
897 status = check_intr_schedule(ehci,
Alan Sternb35c5002013-10-11 22:16:21 -0400898 frame, uframe, qh, &c_mask, tt);
Alan Sternd0ce5c62013-10-11 11:29:13 -0400899 if (status == 0)
Alan Sternb35c5002013-10-11 22:16:21 -0400900 goto got_it;
Alan Stern68335e82009-05-22 17:02:33 -0400901 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
Alan Sternd0ce5c62013-10-11 11:29:13 -0400904 /* qh->ps.bw_period == 0 means every uframe */
905 } else {
Alan Sternb35c5002013-10-11 22:16:21 -0400906 status = check_intr_schedule(ehci, 0, 0, qh, &c_mask, tt);
Alan Sternd0ce5c62013-10-11 11:29:13 -0400907 }
908 if (status)
909 goto done;
Alan Sternb35c5002013-10-11 22:16:21 -0400910
911 got_it:
Alan Sternd0ce5c62013-10-11 11:29:13 -0400912 qh->ps.phase = (qh->ps.period ? ehci->random_frame &
913 (qh->ps.period - 1) : 0);
914 qh->ps.bw_phase = qh->ps.phase & (qh->ps.bw_period - 1);
915 qh->ps.phase_uf = uframe;
916 qh->ps.cs_mask = qh->ps.period ?
917 (c_mask << 8) | (1 << uframe) :
918 QH_SMASK;
919
920 /* reset S-frame and (maybe) C-frame masks */
921 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
922 hw->hw_info2 |= cpu_to_hc32(ehci, qh->ps.cs_mask);
923 reserve_release_intr_bandwidth(ehci, qh, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925done:
926 return status;
927}
928
Geyslan G. Bemee906472016-01-25 22:45:14 -0300929static int intr_submit(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931 struct urb *urb,
932 struct list_head *qtd_list,
Al Viro55016f12005-10-21 03:21:58 -0400933 gfp_t mem_flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934) {
935 unsigned epnum;
936 unsigned long flags;
937 struct ehci_qh *qh;
Alan Sterne9df41c2007-08-08 11:48:02 -0400938 int status;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 struct list_head empty;
940
941 /* get endpoint and transfer/schedule data */
Alan Sterne9df41c2007-08-08 11:48:02 -0400942 epnum = urb->ep->desc.bEndpointAddress;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943
Geyslan G. Bemee906472016-01-25 22:45:14 -0300944 spin_lock_irqsave(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945
Alan Stern541c7d42010-06-22 16:39:10 -0400946 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
Benjamin Herrenschmidt8de98402005-11-25 09:59:46 +1100947 status = -ESHUTDOWN;
Alan Sterne9df41c2007-08-08 11:48:02 -0400948 goto done_not_linked;
Benjamin Herrenschmidt8de98402005-11-25 09:59:46 +1100949 }
Alan Sterne9df41c2007-08-08 11:48:02 -0400950 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
951 if (unlikely(status))
952 goto done_not_linked;
Benjamin Herrenschmidt8de98402005-11-25 09:59:46 +1100953
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 /* get qh and force any scheduling errors */
Geyslan G. Bemee906472016-01-25 22:45:14 -0300955 INIT_LIST_HEAD(&empty);
Alan Sterne9df41c2007-08-08 11:48:02 -0400956 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 if (qh == NULL) {
958 status = -ENOMEM;
959 goto done;
960 }
961 if (qh->qh_state == QH_STATE_IDLE) {
Geyslan G. Bemee906472016-01-25 22:45:14 -0300962 status = qh_schedule(ehci, qh);
963 if (status)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 goto done;
965 }
966
967 /* then queue the urb's tds to the qh */
Alan Sterne9df41c2007-08-08 11:48:02 -0400968 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
Geyslan G. Bemee906472016-01-25 22:45:14 -0300969 BUG_ON(qh == NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970
Alan Sternc1fdb682013-03-22 13:30:43 -0400971 /* stuff into the periodic schedule */
972 if (qh->qh_state == QH_STATE_IDLE) {
973 qh_refresh(ehci, qh);
974 qh_link_periodic(ehci, qh);
Ming Lei9118f9e2013-07-03 22:53:10 +0800975 } else {
976 /* cancel unlink wait for the qh */
977 cancel_unlink_wait_intr(ehci, qh);
Alan Sternc1fdb682013-03-22 13:30:43 -0400978 }
979
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 /* ... update usbfs periodic stats */
981 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
982
983done:
Alan Sterne9df41c2007-08-08 11:48:02 -0400984 if (unlikely(status))
985 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
986done_not_linked:
Geyslan G. Bemee906472016-01-25 22:45:14 -0300987 spin_unlock_irqrestore(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 if (status)
Geyslan G. Bemee906472016-01-25 22:45:14 -0300989 qtd_list_free(ehci, urb, qtd_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990
991 return status;
992}
993
Alan Stern569b3942012-07-11 11:23:00 -0400994static void scan_intr(struct ehci_hcd *ehci)
995{
996 struct ehci_qh *qh;
997
998 list_for_each_entry_safe(qh, ehci->qh_scan_next, &ehci->intr_qh_list,
999 intr_node) {
Alan Stern79bcf7b2013-03-22 13:30:56 -04001000
Alan Stern569b3942012-07-11 11:23:00 -04001001 /* clean any finished work for this qh */
1002 if (!list_empty(&qh->qtd_list)) {
1003 int temp;
1004
1005 /*
1006 * Unlinks could happen here; completion reporting
1007 * drops the lock. That's why ehci->qh_scan_next
1008 * always holds the next qh to scan; if the next qh
1009 * gets unlinked then ehci->qh_scan_next is adjusted
1010 * in qh_unlink_periodic().
1011 */
1012 temp = qh_completions(ehci, qh);
Ming Lei9118f9e2013-07-03 22:53:10 +08001013 if (unlikely(temp))
Alan Stern569b3942012-07-11 11:23:00 -04001014 start_unlink_intr(ehci, qh);
Ming Lei9118f9e2013-07-03 22:53:10 +08001015 else if (unlikely(list_empty(&qh->qtd_list) &&
1016 qh->qh_state == QH_STATE_LINKED))
1017 start_unlink_intr_wait(ehci, qh);
Alan Stern569b3942012-07-11 11:23:00 -04001018 }
1019 }
1020}
1021
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022/*-------------------------------------------------------------------------*/
1023
1024/* ehci_iso_stream ops work with both ITD and SITD */
1025
1026static struct ehci_iso_stream *
Geyslan G. Bemee906472016-01-25 22:45:14 -03001027iso_stream_alloc(gfp_t mem_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028{
1029 struct ehci_iso_stream *stream;
1030
Geyslan G. Bemee2a1d22016-01-25 22:45:18 -03001031 stream = kzalloc(sizeof(*stream), mem_flags);
Geyslan G. Bemee906472016-01-25 22:45:14 -03001032 if (likely(stream != NULL)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 INIT_LIST_HEAD(&stream->td_list);
1034 INIT_LIST_HEAD(&stream->free_list);
Alan Stern91a99b52013-10-11 11:28:52 -04001035 stream->next_uframe = NO_FRAME;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001036 stream->ps.phase = NO_FRAME;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 }
1038 return stream;
1039}
1040
1041static void
Geyslan G. Bemee906472016-01-25 22:45:14 -03001042iso_stream_init(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 struct ehci_hcd *ehci,
1044 struct ehci_iso_stream *stream,
Alan Sternffa02482013-10-11 11:29:03 -04001045 struct urb *urb
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046)
1047{
Geyslan G. Bemee906472016-01-25 22:45:14 -03001048 static const u8 smask_out[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049
Alan Sternffa02482013-10-11 11:29:03 -04001050 struct usb_device *dev = urb->dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051 u32 buf1;
1052 unsigned epnum, maxp;
1053 int is_input;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001054 unsigned tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055
1056 /*
1057 * this might be a "high bandwidth" highspeed endpoint,
1058 * as encoded in the ep descriptor's wMaxPacket field
1059 */
Alan Sternffa02482013-10-11 11:29:03 -04001060 epnum = usb_pipeendpoint(urb->pipe);
1061 is_input = usb_pipein(urb->pipe) ? USB_DIR_IN : 0;
1062 maxp = usb_endpoint_maxp(&urb->ep->desc);
Geyslan G. Bemd078c6e2016-01-25 22:45:19 -03001063 buf1 = is_input ? 1 << 11 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064
1065 /* knows about ITD vs SITD */
1066 if (dev->speed == USB_SPEED_HIGH) {
1067 unsigned multi = hb_mult(maxp);
1068
1069 stream->highspeed = 1;
1070
1071 maxp = max_packet(maxp);
1072 buf1 |= maxp;
1073 maxp *= multi;
1074
Stefan Roese6dbd6822007-05-01 09:29:37 -07001075 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
1076 stream->buf1 = cpu_to_hc32(ehci, buf1);
1077 stream->buf2 = cpu_to_hc32(ehci, multi);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078
1079 /* usbfs wants to report the average usecs per frame tied up
1080 * when transfers on this endpoint are scheduled ...
1081 */
Alan Sternffa02482013-10-11 11:29:03 -04001082 stream->ps.usecs = HS_USECS_ISO(maxp);
1083
Alan Sternd0ce5c62013-10-11 11:29:13 -04001084 /* period for bandwidth allocation */
1085 tmp = min_t(unsigned, EHCI_BANDWIDTH_SIZE,
1086 1 << (urb->ep->desc.bInterval - 1));
1087
1088 /* Allow urb->interval to override */
1089 stream->ps.bw_uperiod = min_t(unsigned, tmp, urb->interval);
1090
1091 stream->uperiod = urb->interval;
1092 stream->ps.period = urb->interval >> 3;
1093 stream->bandwidth = stream->ps.usecs * 8 /
1094 stream->ps.bw_uperiod;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095
1096 } else {
1097 u32 addr;
david-b@pacbell.netd0384202005-08-13 18:44:58 -07001098 int think_time;
Clemens Ladisch469d0222006-01-20 13:49:10 -08001099 int hs_transfers;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100
1101 addr = dev->ttport << 24;
1102 if (!ehci_is_TDI(ehci)
1103 || (dev->tt->hub !=
1104 ehci_to_hcd(ehci)->self.root_hub))
1105 addr |= dev->tt->hub->devnum << 16;
1106 addr |= epnum << 8;
1107 addr |= dev->devnum;
Alan Sternffa02482013-10-11 11:29:03 -04001108 stream->ps.usecs = HS_USECS_ISO(maxp);
david-b@pacbell.netd0384202005-08-13 18:44:58 -07001109 think_time = dev->tt ? dev->tt->think_time : 0;
Alan Sternffa02482013-10-11 11:29:03 -04001110 stream->ps.tt_usecs = NS_TO_US(think_time + usb_calc_bus_time(
david-b@pacbell.netd0384202005-08-13 18:44:58 -07001111 dev->speed, is_input, 1, maxp));
Geyslan G. Bemee906472016-01-25 22:45:14 -03001112 hs_transfers = max(1u, (maxp + 187) / 188);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 if (is_input) {
1114 u32 tmp;
1115
1116 addr |= 1 << 31;
Alan Sternffa02482013-10-11 11:29:03 -04001117 stream->ps.c_usecs = stream->ps.usecs;
1118 stream->ps.usecs = HS_USECS_ISO(1);
Alan Sternd0ce5c62013-10-11 11:29:13 -04001119 stream->ps.cs_mask = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120
Clemens Ladisch469d0222006-01-20 13:49:10 -08001121 /* c-mask as specified in USB 2.0 11.18.4 3.c */
1122 tmp = (1 << (hs_transfers + 2)) - 1;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001123 stream->ps.cs_mask |= tmp << (8 + 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 } else
Alan Sternd0ce5c62013-10-11 11:29:13 -04001125 stream->ps.cs_mask = smask_out[hs_transfers - 1];
Alan Sternffa02482013-10-11 11:29:03 -04001126
Alan Sternd0ce5c62013-10-11 11:29:13 -04001127 /* period for bandwidth allocation */
1128 tmp = min_t(unsigned, EHCI_BANDWIDTH_FRAMES,
1129 1 << (urb->ep->desc.bInterval - 1));
1130
1131 /* Allow urb->interval to override */
1132 stream->ps.bw_period = min_t(unsigned, tmp, urb->interval);
1133 stream->ps.bw_uperiod = stream->ps.bw_period << 3;
1134
1135 stream->ps.period = urb->interval;
1136 stream->uperiod = urb->interval << 3;
Alan Sternffa02482013-10-11 11:29:03 -04001137 stream->bandwidth = (stream->ps.usecs + stream->ps.c_usecs) /
Alan Sternd0ce5c62013-10-11 11:29:13 -04001138 stream->ps.bw_period;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139
Alan Sternd0ce5c62013-10-11 11:29:13 -04001140 /* stream->splits gets created from cs_mask later */
Stefan Roese6dbd6822007-05-01 09:29:37 -07001141 stream->address = cpu_to_hc32(ehci, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
Alan Sternffa02482013-10-11 11:29:03 -04001144 stream->ps.udev = dev;
1145 stream->ps.ep = urb->ep;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146
1147 stream->bEndpointAddress = is_input | epnum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 stream->maxp = maxp;
1149}
1150
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151static struct ehci_iso_stream *
Geyslan G. Bemee906472016-01-25 22:45:14 -03001152iso_stream_find(struct ehci_hcd *ehci, struct urb *urb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153{
1154 unsigned epnum;
1155 struct ehci_iso_stream *stream;
1156 struct usb_host_endpoint *ep;
1157 unsigned long flags;
1158
1159 epnum = usb_pipeendpoint (urb->pipe);
1160 if (usb_pipein(urb->pipe))
1161 ep = urb->dev->ep_in[epnum];
1162 else
1163 ep = urb->dev->ep_out[epnum];
1164
Geyslan G. Bemee906472016-01-25 22:45:14 -03001165 spin_lock_irqsave(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 stream = ep->hcpriv;
1167
Geyslan G. Bemee906472016-01-25 22:45:14 -03001168 if (unlikely(stream == NULL)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001169 stream = iso_stream_alloc(GFP_ATOMIC);
Geyslan G. Bemee906472016-01-25 22:45:14 -03001170 if (likely(stream != NULL)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171 ep->hcpriv = stream;
Alan Sternffa02482013-10-11 11:29:03 -04001172 iso_stream_init(ehci, stream, urb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 }
1174
Clemens Ladisch1082f572010-03-01 17:18:56 +01001175 /* if dev->ep [epnum] is a QH, hw is set */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001176 } else if (unlikely(stream->hw != NULL)) {
1177 ehci_dbg(ehci, "dev %s ep%d%s, not iso??\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 urb->dev->devpath, epnum,
1179 usb_pipein(urb->pipe) ? "in" : "out");
1180 stream = NULL;
1181 }
1182
Geyslan G. Bemee906472016-01-25 22:45:14 -03001183 spin_unlock_irqrestore(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184 return stream;
1185}
1186
1187/*-------------------------------------------------------------------------*/
1188
1189/* ehci_iso_sched ops can be ITD-only or SITD-only */
1190
1191static struct ehci_iso_sched *
Geyslan G. Bemee906472016-01-25 22:45:14 -03001192iso_sched_alloc(unsigned packets, gfp_t mem_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001193{
1194 struct ehci_iso_sched *iso_sched;
Geyslan G. Bemee2a1d22016-01-25 22:45:18 -03001195 int size = sizeof(*iso_sched);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196
Geyslan G. Bemee906472016-01-25 22:45:14 -03001197 size += packets * sizeof(struct ehci_iso_packet);
Eric Sesterhenn80b6ca42006-02-27 21:29:43 +01001198 iso_sched = kzalloc(size, mem_flags);
Geyslan G. Bemee906472016-01-25 22:45:14 -03001199 if (likely(iso_sched != NULL))
1200 INIT_LIST_HEAD(&iso_sched->td_list);
1201
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202 return iso_sched;
1203}
1204
1205static inline void
Stefan Roese6dbd6822007-05-01 09:29:37 -07001206itd_sched_init(
1207 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 struct ehci_iso_sched *iso_sched,
1209 struct ehci_iso_stream *stream,
1210 struct urb *urb
1211)
1212{
1213 unsigned i;
1214 dma_addr_t dma = urb->transfer_dma;
1215
1216 /* how many uframes are needed for these transfers */
Alan Sternffa02482013-10-11 11:29:03 -04001217 iso_sched->span = urb->number_of_packets * stream->uperiod;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218
1219 /* figure out per-uframe itd fields that we'll need later
1220 * when we fit new itds into the schedule.
1221 */
1222 for (i = 0; i < urb->number_of_packets; i++) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03001223 struct ehci_iso_packet *uframe = &iso_sched->packet[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224 unsigned length;
1225 dma_addr_t buf;
1226 u32 trans;
1227
Geyslan G. Bemee906472016-01-25 22:45:14 -03001228 length = urb->iso_frame_desc[i].length;
1229 buf = dma + urb->iso_frame_desc[i].offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
1231 trans = EHCI_ISOC_ACTIVE;
1232 trans |= buf & 0x0fff;
Geyslan G. Bemee906472016-01-25 22:45:14 -03001233 if (unlikely(((i + 1) == urb->number_of_packets))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1235 trans |= EHCI_ITD_IOC;
1236 trans |= length << 16;
Stefan Roese6dbd6822007-05-01 09:29:37 -07001237 uframe->transaction = cpu_to_hc32(ehci, trans);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238
David Brownell77078572005-05-28 10:46:18 -07001239 /* might need to cross a buffer page within a uframe */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 uframe->bufp = (buf & ~(u64)0x0fff);
1241 buf += length;
Geyslan G. Bemee906472016-01-25 22:45:14 -03001242 if (unlikely((uframe->bufp != (buf & ~(u64)0x0fff))))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 uframe->cross = 1;
1244 }
1245}
1246
1247static void
Geyslan G. Bemee906472016-01-25 22:45:14 -03001248iso_sched_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249 struct ehci_iso_stream *stream,
1250 struct ehci_iso_sched *iso_sched
1251)
1252{
1253 if (!iso_sched)
1254 return;
Geyslan G. Bem1ec27802016-01-25 22:45:16 -03001255 /* caller must hold ehci->lock! */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001256 list_splice(&iso_sched->td_list, &stream->free_list);
1257 kfree(iso_sched);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258}
1259
1260static int
Geyslan G. Bemee906472016-01-25 22:45:14 -03001261itd_urb_transaction(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262 struct ehci_iso_stream *stream,
1263 struct ehci_hcd *ehci,
1264 struct urb *urb,
Al Viro55016f12005-10-21 03:21:58 -04001265 gfp_t mem_flags
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266)
1267{
1268 struct ehci_itd *itd;
1269 dma_addr_t itd_dma;
1270 int i;
1271 unsigned num_itds;
1272 struct ehci_iso_sched *sched;
1273 unsigned long flags;
1274
Geyslan G. Bemee906472016-01-25 22:45:14 -03001275 sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
1276 if (unlikely(sched == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277 return -ENOMEM;
1278
Stefan Roese6dbd6822007-05-01 09:29:37 -07001279 itd_sched_init(ehci, sched, stream, urb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280
1281 if (urb->interval < 8)
1282 num_itds = 1 + (sched->span + 7) / 8;
1283 else
1284 num_itds = urb->number_of_packets;
1285
1286 /* allocate/init ITDs */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001287 spin_lock_irqsave(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 for (i = 0; i < num_itds; i++) {
1289
Alan Stern55934eb2012-07-11 11:22:35 -04001290 /*
1291 * Use iTDs from the free list, but not iTDs that may
1292 * still be in use by the hardware.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293 */
Alan Stern55934eb2012-07-11 11:22:35 -04001294 if (likely(!list_empty(&stream->free_list))) {
1295 itd = list_first_entry(&stream->free_list,
Stefan Roese6dbd6822007-05-01 09:29:37 -07001296 struct ehci_itd, itd_list);
Alan Sternf4289072012-07-11 11:23:07 -04001297 if (itd->frame == ehci->now_frame)
Alan Stern55934eb2012-07-11 11:22:35 -04001298 goto alloc_itd;
Geyslan G. Bemee906472016-01-25 22:45:14 -03001299 list_del(&itd->itd_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 itd_dma = itd->itd_dma;
Karsten Wiese3d01f0f2008-02-19 12:31:49 -08001301 } else {
Alan Stern55934eb2012-07-11 11:22:35 -04001302 alloc_itd:
Geyslan G. Bemee906472016-01-25 22:45:14 -03001303 spin_unlock_irqrestore(&ehci->lock, flags);
1304 itd = dma_pool_alloc(ehci->itd_pool, mem_flags,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 &itd_dma);
Geyslan G. Bemee906472016-01-25 22:45:14 -03001306 spin_lock_irqsave(&ehci->lock, flags);
Karsten Wiese3d01f0f2008-02-19 12:31:49 -08001307 if (!itd) {
1308 iso_sched_free(stream, sched);
1309 spin_unlock_irqrestore(&ehci->lock, flags);
1310 return -ENOMEM;
1311 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312 }
1313
Geyslan G. Bemee2a1d22016-01-25 22:45:18 -03001314 memset(itd, 0, sizeof(*itd));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315 itd->itd_dma = itd_dma;
Alan Stern91a99b52013-10-11 11:28:52 -04001316 itd->frame = NO_FRAME;
Geyslan G. Bemee906472016-01-25 22:45:14 -03001317 list_add(&itd->itd_list, &sched->td_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318 }
Geyslan G. Bemee906472016-01-25 22:45:14 -03001319 spin_unlock_irqrestore(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320
1321 /* temporarily store schedule info in hcpriv */
1322 urb->hcpriv = sched;
1323 urb->error_count = 0;
1324 return 0;
1325}
1326
1327/*-------------------------------------------------------------------------*/
1328
Alan Sternd0ce5c62013-10-11 11:29:13 -04001329static void reserve_release_iso_bandwidth(struct ehci_hcd *ehci,
1330 struct ehci_iso_stream *stream, int sign)
1331{
1332 unsigned uframe;
1333 unsigned i, j;
1334 unsigned s_mask, c_mask, m;
1335 int usecs = stream->ps.usecs;
1336 int c_usecs = stream->ps.c_usecs;
Alan Sternb35c5002013-10-11 22:16:21 -04001337 int tt_usecs = stream->ps.tt_usecs;
1338 struct ehci_tt *tt;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001339
1340 if (stream->ps.phase == NO_FRAME) /* Bandwidth wasn't reserved */
1341 return;
1342 uframe = stream->ps.bw_phase << 3;
1343
1344 bandwidth_dbg(ehci, sign, "iso", &stream->ps);
1345
1346 if (sign < 0) { /* Release bandwidth */
1347 usecs = -usecs;
1348 c_usecs = -c_usecs;
Alan Sternb35c5002013-10-11 22:16:21 -04001349 tt_usecs = -tt_usecs;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001350 }
1351
1352 if (!stream->splits) { /* High speed */
1353 for (i = uframe + stream->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE;
1354 i += stream->ps.bw_uperiod)
1355 ehci->bandwidth[i] += usecs;
1356
1357 } else { /* Full speed */
1358 s_mask = stream->ps.cs_mask;
1359 c_mask = s_mask >> 8;
1360
1361 /* NOTE: adjustment needed for frame overflow */
1362 for (i = uframe; i < EHCI_BANDWIDTH_SIZE;
1363 i += stream->ps.bw_uperiod) {
1364 for ((j = stream->ps.phase_uf, m = 1 << j); j < 8;
1365 (++j, m <<= 1)) {
1366 if (s_mask & m)
1367 ehci->bandwidth[i+j] += usecs;
1368 else if (c_mask & m)
1369 ehci->bandwidth[i+j] += c_usecs;
1370 }
1371 }
Alan Sternb35c5002013-10-11 22:16:21 -04001372
1373 tt = find_tt(stream->ps.udev);
1374 if (sign > 0)
1375 list_add_tail(&stream->ps.ps_list, &tt->ps_list);
1376 else
1377 list_del(&stream->ps.ps_list);
1378
1379 for (i = uframe >> 3; i < EHCI_BANDWIDTH_FRAMES;
1380 i += stream->ps.bw_period)
1381 tt->bandwidth[i] += tt_usecs;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001382 }
1383}
1384
Linus Torvalds1da177e2005-04-16 15:20:36 -07001385static inline int
Geyslan G. Bemee906472016-01-25 22:45:14 -03001386itd_slot_ok(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387 struct ehci_hcd *ehci,
Alan Sternd0ce5c62013-10-11 11:29:13 -04001388 struct ehci_iso_stream *stream,
1389 unsigned uframe
Linus Torvalds1da177e2005-04-16 15:20:36 -07001390)
1391{
Alan Sternd0ce5c62013-10-11 11:29:13 -04001392 unsigned usecs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393
Alan Sternd0ce5c62013-10-11 11:29:13 -04001394 /* convert "usecs we need" to "max already claimed" */
1395 usecs = ehci->uframe_periodic_max - stream->ps.usecs;
1396
1397 for (uframe &= stream->ps.bw_uperiod - 1; uframe < EHCI_BANDWIDTH_SIZE;
1398 uframe += stream->ps.bw_uperiod) {
1399 if (ehci->bandwidth[uframe] > usecs)
1400 return 0;
1401 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402 return 1;
1403}
1404
1405static inline int
Geyslan G. Bemee906472016-01-25 22:45:14 -03001406sitd_slot_ok(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408 struct ehci_iso_stream *stream,
Alan Sternd0ce5c62013-10-11 11:29:13 -04001409 unsigned uframe,
Alan Sternb35c5002013-10-11 22:16:21 -04001410 struct ehci_iso_sched *sched,
1411 struct ehci_tt *tt
Linus Torvalds1da177e2005-04-16 15:20:36 -07001412)
1413{
Alan Sternd0ce5c62013-10-11 11:29:13 -04001414 unsigned mask, tmp;
1415 unsigned frame, uf;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416
Alan Sternd0ce5c62013-10-11 11:29:13 -04001417 mask = stream->ps.cs_mask << (uframe & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418
Alan Stern8c05dc52013-10-11 11:28:31 -04001419 /* for OUT, don't wrap SSPLIT into H-microframe 7 */
Alan Sternd0ce5c62013-10-11 11:29:13 -04001420 if (((stream->ps.cs_mask & 0xff) << (uframe & 7)) >= (1 << 7))
Alan Stern8c05dc52013-10-11 11:28:31 -04001421 return 0;
1422
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423 /* for IN, don't wrap CSPLIT into the next frame */
1424 if (mask & ~0xffff)
1425 return 0;
1426
Alan Stern65b8e5c2012-05-14 13:47:20 -04001427 /* check bandwidth */
Alan Sternd0ce5c62013-10-11 11:29:13 -04001428 uframe &= stream->ps.bw_uperiod - 1;
Alan Stern65b8e5c2012-05-14 13:47:20 -04001429 frame = uframe >> 3;
1430
1431#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1432 /* The tt's fullspeed bus bandwidth must be available.
1433 * tt_available scheduling guarantees 10+% for control/bulk.
1434 */
1435 uf = uframe & 7;
Alan Sternb35c5002013-10-11 22:16:21 -04001436 if (!tt_available(ehci, &stream->ps, tt, frame, uf))
Alan Stern65b8e5c2012-05-14 13:47:20 -04001437 return 0;
1438#else
1439 /* tt must be idle for start(s), any gap, and csplit.
1440 * assume scheduling slop leaves 10+% for control/bulk.
1441 */
Alan Sternd0ce5c62013-10-11 11:29:13 -04001442 if (!tt_no_collision(ehci, stream->ps.bw_period,
Alan Sternffa02482013-10-11 11:29:03 -04001443 stream->ps.udev, frame, mask))
Alan Stern65b8e5c2012-05-14 13:47:20 -04001444 return 0;
1445#endif
1446
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447 do {
Alan Sternd0ce5c62013-10-11 11:29:13 -04001448 unsigned max_used;
1449 unsigned i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451 /* check starts (OUT uses more than one) */
Alan Sternd0ce5c62013-10-11 11:29:13 -04001452 uf = uframe;
Alan Sternffa02482013-10-11 11:29:03 -04001453 max_used = ehci->uframe_periodic_max - stream->ps.usecs;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001454 for (tmp = stream->ps.cs_mask & 0xff; tmp; tmp >>= 1, uf++) {
1455 if (ehci->bandwidth[uf] > max_used)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456 return 0;
1457 }
1458
1459 /* for IN, check CSPLIT */
Alan Sternffa02482013-10-11 11:29:03 -04001460 if (stream->ps.c_usecs) {
Alan Sternffa02482013-10-11 11:29:03 -04001461 max_used = ehci->uframe_periodic_max -
1462 stream->ps.c_usecs;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001463 uf = uframe & ~7;
1464 tmp = 1 << (2+8);
1465 for (i = (uframe & 7) + 2; i < 8; (++i, tmp <<= 1)) {
1466 if ((stream->ps.cs_mask & tmp) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467 continue;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001468 if (ehci->bandwidth[uf+i] > max_used)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469 return 0;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001470 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471 }
1472
Alan Sternd0ce5c62013-10-11 11:29:13 -04001473 uframe += stream->ps.bw_uperiod;
1474 } while (uframe < EHCI_BANDWIDTH_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475
Alan Sternd0ce5c62013-10-11 11:29:13 -04001476 stream->ps.cs_mask <<= uframe & 7;
1477 stream->splits = cpu_to_hc32(ehci, stream->ps.cs_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478 return 1;
1479}
1480
1481/*
1482 * This scheduler plans almost as far into the future as it has actual
1483 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1484 * "as small as possible" to be cache-friendlier.) That limits the size
1485 * transfers you can stream reliably; avoid more than 64 msec per urb.
1486 * Also avoid queue depths of less than ehci's worst irq latency (affected
1487 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1488 * and other factors); or more than about 230 msec total (for portability,
1489 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1490 */
1491
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492static int
Geyslan G. Bemee906472016-01-25 22:45:14 -03001493iso_stream_schedule(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494 struct ehci_hcd *ehci,
1495 struct urb *urb,
1496 struct ehci_iso_stream *stream
1497)
1498{
Alan Stern46c73d12013-09-03 13:59:03 -04001499 u32 now, base, next, start, period, span, now2;
1500 u32 wrap = 0, skip = 0;
1501 int status = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 unsigned mod = ehci->periodic_size << 3;
1503 struct ehci_iso_sched *sched = urb->hcpriv;
Alan Stern46c73d12013-09-03 13:59:03 -04001504 bool empty = list_empty(&stream->td_list);
Alan Sterna393a802013-10-11 11:29:36 -04001505 bool new_stream = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506
Alan Sternd0ce5c62013-10-11 11:29:13 -04001507 period = stream->uperiod;
Alan Sternffda0802010-07-14 11:03:46 -04001508 span = sched->span;
Alan Sternd0ce5c62013-10-11 11:29:13 -04001509 if (!stream->highspeed)
Alan Sternffda0802010-07-14 11:03:46 -04001510 span <<= 3;
Alan Sternffda0802010-07-14 11:03:46 -04001511
Alan Sternd0ce5c62013-10-11 11:29:13 -04001512 /* Start a new isochronous stream? */
Alan Stern46c73d12013-09-03 13:59:03 -04001513 if (unlikely(empty && !hcd_periodic_completion_in_progress(
1514 ehci_to_hcd(ehci), urb->ep))) {
Alan Sterne4e18cb2013-09-03 13:58:59 -04001515
Alan Sternd0ce5c62013-10-11 11:29:13 -04001516 /* Schedule the endpoint */
1517 if (stream->ps.phase == NO_FRAME) {
1518 int done = 0;
Alan Sternb35c5002013-10-11 22:16:21 -04001519 struct ehci_tt *tt = find_tt(stream->ps.udev);
1520
1521 if (IS_ERR(tt)) {
1522 status = PTR_ERR(tt);
1523 goto fail;
1524 }
1525 compute_tt_budget(ehci->tt_budget, tt);
Alan Sterne4e18cb2013-09-03 13:58:59 -04001526
Alan Sterna393a802013-10-11 11:29:36 -04001527 start = ((-(++ehci->random_frame)) << 3) & (period - 1);
Alan Sternd0ce5c62013-10-11 11:29:13 -04001528
1529 /* find a uframe slot with enough bandwidth.
1530 * Early uframes are more precious because full-speed
1531 * iso IN transfers can't use late uframes,
1532 * and therefore they should be allocated last.
1533 */
1534 next = start;
1535 start += period;
1536 do {
1537 start--;
1538 /* check schedule: enough space? */
1539 if (stream->highspeed) {
1540 if (itd_slot_ok(ehci, stream, start))
1541 done = 1;
1542 } else {
1543 if ((start % 8) >= 6)
1544 continue;
1545 if (sitd_slot_ok(ehci, stream, start,
Alan Sternb35c5002013-10-11 22:16:21 -04001546 sched, tt))
Alan Sternd0ce5c62013-10-11 11:29:13 -04001547 done = 1;
1548 }
1549 } while (start > next && !done);
1550
1551 /* no room in the schedule */
1552 if (!done) {
1553 ehci_dbg(ehci, "iso sched full %p", urb);
1554 status = -ENOSPC;
1555 goto fail;
Alan Sterne4e18cb2013-09-03 13:58:59 -04001556 }
Alan Sternd0ce5c62013-10-11 11:29:13 -04001557 stream->ps.phase = (start >> 3) &
1558 (stream->ps.period - 1);
1559 stream->ps.bw_phase = stream->ps.phase &
1560 (stream->ps.bw_period - 1);
1561 stream->ps.phase_uf = start & 7;
1562 reserve_release_iso_bandwidth(ehci, stream, 1);
1563 }
Alan Sterne4e18cb2013-09-03 13:58:59 -04001564
Alan Sternd0ce5c62013-10-11 11:29:13 -04001565 /* New stream is already scheduled; use the upcoming slot */
1566 else {
1567 start = (stream->ps.phase << 3) + stream->ps.phase_uf;
Alan Sterne4e18cb2013-09-03 13:58:59 -04001568 }
Alan Stern46c73d12013-09-03 13:59:03 -04001569
Alan Sterna393a802013-10-11 11:29:36 -04001570 stream->next_uframe = start;
1571 new_stream = true;
Alan Sterne4e18cb2013-09-03 13:58:59 -04001572 }
1573
Alan Sterna393a802013-10-11 11:29:36 -04001574 now = ehci_read_frame_index(ehci) & (mod - 1);
1575
1576 /* Take the isochronous scheduling threshold into account */
1577 if (ehci->i_thresh)
1578 next = now + ehci->i_thresh; /* uframe cache */
1579 else
1580 next = (now + 2 + 7) & ~0x07; /* full frame cache */
1581
Alan Stern6d892522014-12-04 10:21:56 -05001582 /* If needed, initialize last_iso_frame so that this URB will be seen */
1583 if (ehci->isoc_count == 0)
1584 ehci->last_iso_frame = now >> 3;
1585
Alan Sterna393a802013-10-11 11:29:36 -04001586 /*
1587 * Use ehci->last_iso_frame as the base. There can't be any
1588 * TDs scheduled for earlier than that.
1589 */
1590 base = ehci->last_iso_frame << 3;
1591 next = (next - base) & (mod - 1);
1592 start = (stream->next_uframe - base) & (mod - 1);
1593
1594 if (unlikely(new_stream))
1595 goto do_ASAP;
1596
Alan Sterne4e18cb2013-09-03 13:58:59 -04001597 /*
Alan Sternd0ce5c62013-10-11 11:29:13 -04001598 * Typical case: reuse current schedule, stream may still be active.
Alan Sternb40e43f2008-05-20 16:59:10 -04001599 * Hopefully there are no gaps from the host falling behind
Alan Stern4005ad42012-10-01 10:32:01 -04001600 * (irq delays etc). If there are, the behavior depends on
1601 * whether URB_ISO_ASAP is set.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602 */
Alan Stern46c73d12013-09-03 13:59:03 -04001603 now2 = (now - base) & (mod - 1);
Sarah Sharpdccd5742009-10-27 10:55:05 -07001604
Alan Sternc401e7b2014-12-04 10:22:57 -05001605 /* Is the schedule about to wrap around? */
Alan Stern46c73d12013-09-03 13:59:03 -04001606 if (unlikely(!empty && start < period)) {
Alan Sternc401e7b2014-12-04 10:22:57 -05001607 ehci_dbg(ehci, "request %p would overflow (%u-%u < %u mod %u)\n",
Alan Stern46c73d12013-09-03 13:59:03 -04001608 urb, stream->next_uframe, base, period, mod);
Alan Sternc401e7b2014-12-04 10:22:57 -05001609 status = -EFBIG;
Alan Stern46c73d12013-09-03 13:59:03 -04001610 goto fail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611 }
1612
Alan Stern46c73d12013-09-03 13:59:03 -04001613 /* Is the next packet scheduled after the base time? */
1614 if (likely(!empty || start <= now2 + period)) {
1615
1616 /* URB_ISO_ASAP: make sure that start >= next */
1617 if (unlikely(start < next &&
1618 (urb->transfer_flags & URB_ISO_ASAP)))
1619 goto do_ASAP;
1620
1621 /* Otherwise use start, if it's not in the past */
1622 if (likely(start >= now2))
1623 goto use_start;
1624
1625 /* Otherwise we got an underrun while the queue was empty */
1626 } else {
1627 if (urb->transfer_flags & URB_ISO_ASAP)
1628 goto do_ASAP;
1629 wrap = mod;
1630 now2 += mod;
1631 }
1632
1633 /* How many uframes and packets do we need to skip? */
1634 skip = (now2 - start + period - 1) & -period;
1635 if (skip >= span) { /* Entirely in the past? */
1636 ehci_dbg(ehci, "iso underrun %p (%u+%u < %u) [%u]\n",
1637 urb, start + base, span - period, now2 + base,
1638 base);
1639
1640 /* Try to keep the last TD intact for scanning later */
1641 skip = span - period;
1642
1643 /* Will it come before the current scan position? */
1644 if (empty) {
1645 skip = span; /* Skip the entire URB */
1646 status = 1; /* and give it back immediately */
1647 iso_sched_free(stream, sched);
1648 sched = NULL;
1649 }
1650 }
1651 urb->error_count = skip / period;
1652 if (sched)
1653 sched->first_packet = urb->error_count;
1654 goto use_start;
1655
1656 do_ASAP:
1657 /* Use the first slot after "next" */
1658 start = next + ((start - next) & (period - 1));
1659
1660 use_start:
Alan Stern1fb2e052010-07-14 11:03:53 -04001661 /* Tried to schedule too far into the future? */
Alan Stern46c73d12013-09-03 13:59:03 -04001662 if (unlikely(start + span - period >= mod + wrap)) {
Alan Sternc3ee9b72012-09-28 16:01:23 -04001663 ehci_dbg(ehci, "request %p would overflow (%u+%u >= %u)\n",
Alan Stern46c73d12013-09-03 13:59:03 -04001664 urb, start, span - period, mod + wrap);
Alan Stern1fb2e052010-07-14 11:03:53 -04001665 status = -EFBIG;
1666 goto fail;
1667 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668
Alan Stern46c73d12013-09-03 13:59:03 -04001669 start += base;
1670 stream->next_uframe = (start + skip) & (mod - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672 /* report high speed start in uframes; full speed, in frames */
Alan Stern46c73d12013-09-03 13:59:03 -04001673 urb->start_frame = start & (mod - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674 if (!stream->highspeed)
1675 urb->start_frame >>= 3;
Alan Stern46c73d12013-09-03 13:59:03 -04001676 return status;
Alan Stern1fb2e052010-07-14 11:03:53 -04001677
1678 fail:
1679 iso_sched_free(stream, sched);
1680 urb->hcpriv = NULL;
1681 return status;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682}
1683
1684/*-------------------------------------------------------------------------*/
1685
1686static inline void
Stefan Roese6dbd6822007-05-01 09:29:37 -07001687itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1688 struct ehci_itd *itd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689{
1690 int i;
1691
David Brownell77078572005-05-28 10:46:18 -07001692 /* it's been recently zeroed */
Stefan Roese6dbd6822007-05-01 09:29:37 -07001693 itd->hw_next = EHCI_LIST_END(ehci);
Geyslan G. Bemee906472016-01-25 22:45:14 -03001694 itd->hw_bufp[0] = stream->buf0;
1695 itd->hw_bufp[1] = stream->buf1;
1696 itd->hw_bufp[2] = stream->buf2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697
1698 for (i = 0; i < 8; i++)
1699 itd->index[i] = -1;
1700
1701 /* All other fields are filled when scheduling */
1702}
1703
1704static inline void
Stefan Roese6dbd6822007-05-01 09:29:37 -07001705itd_patch(
1706 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707 struct ehci_itd *itd,
1708 struct ehci_iso_sched *iso_sched,
1709 unsigned index,
David Brownell77078572005-05-28 10:46:18 -07001710 u16 uframe
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711)
1712{
Geyslan G. Bemee906472016-01-25 22:45:14 -03001713 struct ehci_iso_packet *uf = &iso_sched->packet[index];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714 unsigned pg = itd->pg;
1715
Geyslan G. Bem1ec27802016-01-25 22:45:16 -03001716 /* BUG_ON(pg == 6 && uf->cross); */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717
1718 uframe &= 0x07;
Geyslan G. Bemee906472016-01-25 22:45:14 -03001719 itd->index[uframe] = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001720
Stefan Roese6dbd6822007-05-01 09:29:37 -07001721 itd->hw_transaction[uframe] = uf->transaction;
1722 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1723 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1724 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725
1726 /* iso_frame_desc[].offset must be strictly increasing */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001727 if (unlikely(uf->cross)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728 u64 bufp = uf->bufp + 4096;
Stefan Roese6dbd6822007-05-01 09:29:37 -07001729
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730 itd->pg = ++pg;
Stefan Roese6dbd6822007-05-01 09:29:37 -07001731 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1732 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001733 }
1734}
1735
1736static inline void
Geyslan G. Bemee906472016-01-25 22:45:14 -03001737itd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738{
Clemens Ladisch92bc3642010-03-01 09:12:50 +01001739 union ehci_shadow *prev = &ehci->pshadow[frame];
1740 __hc32 *hw_p = &ehci->periodic[frame];
1741 union ehci_shadow here = *prev;
1742 __hc32 type = 0;
1743
1744 /* skip any iso nodes which might belong to previous microframes */
1745 while (here.ptr) {
1746 type = Q_NEXT_TYPE(ehci, *hw_p);
1747 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1748 break;
1749 prev = periodic_next_shadow(ehci, prev, type);
1750 hw_p = shadow_next_periodic(ehci, &here, type);
1751 here = *prev;
1752 }
1753
1754 itd->itd_next = here;
1755 itd->hw_next = *hw_p;
1756 prev->itd = itd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001757 itd->frame = frame;
Geyslan G. Bemee906472016-01-25 22:45:14 -03001758 wmb();
Clemens Ladisch92bc3642010-03-01 09:12:50 +01001759 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001760}
1761
1762/* fit urb's itds into the selected schedule slot; activate as needed */
Alan Sternb015cb72012-07-11 11:22:10 -04001763static void itd_link_urb(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764 struct ehci_hcd *ehci,
1765 struct urb *urb,
1766 unsigned mod,
1767 struct ehci_iso_stream *stream
1768)
1769{
David Brownell77078572005-05-28 10:46:18 -07001770 int packet;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 unsigned next_uframe, uframe, frame;
1772 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1773 struct ehci_itd *itd;
1774
Alan Sternbccbefa2010-07-14 11:03:36 -04001775 next_uframe = stream->next_uframe & (mod - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776
Geyslan G. Bemee906472016-01-25 22:45:14 -03001777 if (unlikely(list_empty(&stream->td_list)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778 ehci_to_hcd(ehci)->self.bandwidth_allocated
1779 += stream->bandwidth;
Alex He05570292010-12-07 10:10:08 +08001780
1781 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
Andiry Xuad935622011-03-01 14:57:05 +08001782 if (ehci->amd_pll_fix == 1)
1783 usb_amd_quirk_pll_disable();
Alex He05570292010-12-07 10:10:08 +08001784 }
1785
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1787
1788 /* fill iTDs uframe by uframe */
Alan Stern46c73d12013-09-03 13:59:03 -04001789 for (packet = iso_sched->first_packet, itd = NULL;
1790 packet < urb->number_of_packets;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 if (itd == NULL) {
1792 /* ASSERT: we have all necessary itds */
Geyslan G. Bem1ec27802016-01-25 22:45:16 -03001793 /* BUG_ON(list_empty(&iso_sched->td_list)); */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001794
1795 /* ASSERT: no itds for this endpoint in this uframe */
1796
Geyslan G. Bemee906472016-01-25 22:45:14 -03001797 itd = list_entry(iso_sched->td_list.next,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 struct ehci_itd, itd_list);
Geyslan G. Bemee906472016-01-25 22:45:14 -03001799 list_move_tail(&itd->itd_list, &stream->td_list);
Alan Stern8c5bf7b2012-07-11 11:22:39 -04001800 itd->stream = stream;
Karsten Wiese508db8c2009-02-26 01:47:48 +01001801 itd->urb = urb;
Geyslan G. Bemee906472016-01-25 22:45:14 -03001802 itd_init(ehci, stream, itd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803 }
1804
1805 uframe = next_uframe & 0x07;
1806 frame = next_uframe >> 3;
1807
Stefan Roese6dbd6822007-05-01 09:29:37 -07001808 itd_patch(ehci, itd, iso_sched, packet, uframe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001809
Alan Sternffa02482013-10-11 11:29:03 -04001810 next_uframe += stream->uperiod;
Alan Sternbccbefa2010-07-14 11:03:36 -04001811 next_uframe &= mod - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001812 packet++;
1813
1814 /* link completed itds into the schedule */
1815 if (((next_uframe >> 3) != frame)
1816 || packet == urb->number_of_packets) {
Alan Sternbccbefa2010-07-14 11:03:36 -04001817 itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818 itd = NULL;
1819 }
1820 }
1821 stream->next_uframe = next_uframe;
1822
1823 /* don't need that schedule data any more */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001824 iso_sched_free(stream, iso_sched);
Alan Stern2656a9a2012-11-08 10:17:01 -05001825 urb->hcpriv = stream;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001826
Alan Stern569b3942012-07-11 11:23:00 -04001827 ++ehci->isoc_count;
Alan Sternb015cb72012-07-11 11:22:10 -04001828 enable_periodic(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829}
1830
1831#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1832
David Brownell30bf54e2007-12-16 22:37:40 -08001833/* Process and recycle a completed ITD. Return true iff its urb completed,
1834 * and hence its completion callback probably added things to the hardware
1835 * schedule.
1836 *
1837 * Note that we carefully avoid recycling this descriptor until after any
1838 * completion callback runs, so that it won't be reused quickly. That is,
1839 * assuming (a) no more than two urbs per frame on this endpoint, and also
1840 * (b) only this endpoint's completions submit URBs. It seems some silicon
1841 * corrupts things if you reuse completed descriptors very quickly...
1842 */
Alan Sternf4289072012-07-11 11:23:07 -04001843static bool itd_complete(struct ehci_hcd *ehci, struct ehci_itd *itd)
1844{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001845 struct urb *urb = itd->urb;
1846 struct usb_iso_packet_descriptor *desc;
1847 u32 t;
1848 unsigned uframe;
1849 int urb_index = -1;
1850 struct ehci_iso_stream *stream = itd->stream;
1851 struct usb_device *dev;
Alan Sternf4289072012-07-11 11:23:07 -04001852 bool retval = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853
1854 /* for each uframe with a packet */
1855 for (uframe = 0; uframe < 8; uframe++) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03001856 if (likely(itd->index[uframe] == -1))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857 continue;
1858 urb_index = itd->index[uframe];
Geyslan G. Bemee906472016-01-25 22:45:14 -03001859 desc = &urb->iso_frame_desc[urb_index];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860
Geyslan G. Bemee906472016-01-25 22:45:14 -03001861 t = hc32_to_cpup(ehci, &itd->hw_transaction[uframe]);
1862 itd->hw_transaction[uframe] = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863
1864 /* report transfer status */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001865 if (unlikely(t & ISO_ERRS)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866 urb->error_count++;
1867 if (t & EHCI_ISOC_BUF_ERR)
Geyslan G. Bemee906472016-01-25 22:45:14 -03001868 desc->status = usb_pipein(urb->pipe)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869 ? -ENOSR /* hc couldn't read */
1870 : -ECOMM; /* hc couldn't write */
1871 else if (t & EHCI_ISOC_BABBLE)
1872 desc->status = -EOVERFLOW;
1873 else /* (t & EHCI_ISOC_XACTERR) */
1874 desc->status = -EPROTO;
1875
1876 /* HC need not update length with this error */
Alan Sternec6d67e2009-06-29 14:34:59 -04001877 if (!(t & EHCI_ISOC_BABBLE)) {
1878 desc->actual_length = EHCI_ITD_LENGTH(t);
1879 urb->actual_length += desc->actual_length;
1880 }
Geyslan G. Bemee906472016-01-25 22:45:14 -03001881 } else if (likely((t & EHCI_ISOC_ACTIVE) == 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882 desc->status = 0;
Alan Sternec6d67e2009-06-29 14:34:59 -04001883 desc->actual_length = EHCI_ITD_LENGTH(t);
1884 urb->actual_length += desc->actual_length;
Alan Sternb40e43f2008-05-20 16:59:10 -04001885 } else {
1886 /* URB was too late */
Alan Stern4005ad42012-10-01 10:32:01 -04001887 urb->error_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001888 }
1889 }
1890
Linus Torvalds1da177e2005-04-16 15:20:36 -07001891 /* handle completion now? */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001892 if (likely((urb_index + 1) != urb->number_of_packets))
David Brownell30bf54e2007-12-16 22:37:40 -08001893 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894
Geyslan G. Bem1ec27802016-01-25 22:45:16 -03001895 /*
1896 * ASSERT: it's really the last itd for this urb
1897 * list_for_each_entry (itd, &stream->td_list, itd_list)
1898 * BUG_ON(itd->urb == urb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899 */
1900
David Brownellaa16ca32007-12-30 23:45:19 -08001901 /* give urb back to the driver; completion often (re)submits */
Alan Stern6a8e87b2006-01-19 10:46:27 -05001902 dev = urb->dev;
Alan Stern14c04c02007-08-24 15:40:19 -04001903 ehci_urb_done(ehci, urb, 0);
David Brownell30bf54e2007-12-16 22:37:40 -08001904 retval = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 urb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001906
Alan Stern569b3942012-07-11 11:23:00 -04001907 --ehci->isoc_count;
1908 disable_periodic(ehci);
1909
1910 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
Alex He05570292010-12-07 10:10:08 +08001911 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
Andiry Xuad935622011-03-01 14:57:05 +08001912 if (ehci->amd_pll_fix == 1)
1913 usb_amd_quirk_pll_enable();
Alex He05570292010-12-07 10:10:08 +08001914 }
1915
Xenia Ragiadakoufea26ef2013-08-29 11:45:10 +03001916 if (unlikely(list_is_singular(&stream->td_list)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917 ehci_to_hcd(ehci)->self.bandwidth_allocated
1918 -= stream->bandwidth;
Karsten Wiese9aa09d22009-02-08 16:07:58 -08001919
David Brownell30bf54e2007-12-16 22:37:40 -08001920done:
David Brownell30bf54e2007-12-16 22:37:40 -08001921 itd->urb = NULL;
Alan Stern55934eb2012-07-11 11:22:35 -04001922
1923 /* Add to the end of the free list for later reuse */
1924 list_move_tail(&itd->itd_list, &stream->free_list);
1925
1926 /* Recycle the iTDs when the pipeline is empty (ep no longer in use) */
1927 if (list_empty(&stream->td_list)) {
1928 list_splice_tail_init(&stream->free_list,
1929 &ehci->cached_itd_list);
1930 start_free_itds(ehci);
Karsten Wiese9aa09d22009-02-08 16:07:58 -08001931 }
Alan Stern55934eb2012-07-11 11:22:35 -04001932
David Brownell30bf54e2007-12-16 22:37:40 -08001933 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934}
1935
1936/*-------------------------------------------------------------------------*/
1937
Geyslan G. Bemee906472016-01-25 22:45:14 -03001938static int itd_submit(struct ehci_hcd *ehci, struct urb *urb,
Al Viro55016f12005-10-21 03:21:58 -04001939 gfp_t mem_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940{
1941 int status = -EINVAL;
1942 unsigned long flags;
1943 struct ehci_iso_stream *stream;
1944
1945 /* Get iso_stream head */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001946 stream = iso_stream_find(ehci, urb);
1947 if (unlikely(stream == NULL)) {
1948 ehci_dbg(ehci, "can't get iso stream\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001949 return -ENOMEM;
1950 }
Alan Sternffa02482013-10-11 11:29:03 -04001951 if (unlikely(urb->interval != stream->uperiod)) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03001952 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
Alan Sternffa02482013-10-11 11:29:03 -04001953 stream->uperiod, urb->interval);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954 goto done;
1955 }
1956
1957#ifdef EHCI_URB_TRACE
Geyslan G. Bemee906472016-01-25 22:45:14 -03001958 ehci_dbg(ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001959 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
Harvey Harrison441b62c2008-03-03 16:08:34 -08001960 __func__, urb->dev->devpath, urb,
Geyslan G. Bemee906472016-01-25 22:45:14 -03001961 usb_pipeendpoint(urb->pipe),
1962 usb_pipein(urb->pipe) ? "in" : "out",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001963 urb->transfer_buffer_length,
1964 urb->number_of_packets, urb->interval,
1965 stream);
1966#endif
1967
1968 /* allocate ITDs w/o locking anything */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001969 status = itd_urb_transaction(stream, ehci, urb, mem_flags);
1970 if (unlikely(status < 0)) {
1971 ehci_dbg(ehci, "can't init itds\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972 goto done;
1973 }
1974
1975 /* schedule ... need to lock */
Geyslan G. Bemee906472016-01-25 22:45:14 -03001976 spin_lock_irqsave(&ehci->lock, flags);
Alan Stern541c7d42010-06-22 16:39:10 -04001977 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
Benjamin Herrenschmidt8de98402005-11-25 09:59:46 +11001978 status = -ESHUTDOWN;
Alan Sterne9df41c2007-08-08 11:48:02 -04001979 goto done_not_linked;
1980 }
1981 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1982 if (unlikely(status))
1983 goto done_not_linked;
1984 status = iso_stream_schedule(ehci, urb, stream);
Alan Stern46c73d12013-09-03 13:59:03 -04001985 if (likely(status == 0)) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03001986 itd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
Alan Stern46c73d12013-09-03 13:59:03 -04001987 } else if (status > 0) {
1988 status = 0;
1989 ehci_urb_done(ehci, urb, 0);
1990 } else {
Alan Sterne9df41c2007-08-08 11:48:02 -04001991 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
Alan Stern46c73d12013-09-03 13:59:03 -04001992 }
Alan Stern8c5bf7b2012-07-11 11:22:39 -04001993 done_not_linked:
Geyslan G. Bemee906472016-01-25 22:45:14 -03001994 spin_unlock_irqrestore(&ehci->lock, flags);
Alan Stern8c5bf7b2012-07-11 11:22:39 -04001995 done:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001996 return status;
1997}
1998
Linus Torvalds1da177e2005-04-16 15:20:36 -07001999/*-------------------------------------------------------------------------*/
2000
2001/*
2002 * "Split ISO TDs" ... used for USB 1.1 devices going through the
2003 * TTs in USB 2.0 hubs. These need microframe scheduling.
2004 */
2005
2006static inline void
Stefan Roese6dbd6822007-05-01 09:29:37 -07002007sitd_sched_init(
2008 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002009 struct ehci_iso_sched *iso_sched,
2010 struct ehci_iso_stream *stream,
2011 struct urb *urb
2012)
2013{
2014 unsigned i;
2015 dma_addr_t dma = urb->transfer_dma;
2016
2017 /* how many frames are needed for these transfers */
Alan Sternffa02482013-10-11 11:29:03 -04002018 iso_sched->span = urb->number_of_packets * stream->ps.period;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019
2020 /* figure out per-frame sitd fields that we'll need later
2021 * when we fit new sitds into the schedule.
2022 */
2023 for (i = 0; i < urb->number_of_packets; i++) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03002024 struct ehci_iso_packet *packet = &iso_sched->packet[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025 unsigned length;
2026 dma_addr_t buf;
2027 u32 trans;
2028
Geyslan G. Bemee906472016-01-25 22:45:14 -03002029 length = urb->iso_frame_desc[i].length & 0x03ff;
2030 buf = dma + urb->iso_frame_desc[i].offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031
2032 trans = SITD_STS_ACTIVE;
2033 if (((i + 1) == urb->number_of_packets)
2034 && !(urb->transfer_flags & URB_NO_INTERRUPT))
2035 trans |= SITD_IOC;
2036 trans |= length << 16;
Stefan Roese6dbd6822007-05-01 09:29:37 -07002037 packet->transaction = cpu_to_hc32(ehci, trans);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002038
2039 /* might need to cross a buffer page within a td */
2040 packet->bufp = buf;
2041 packet->buf1 = (buf + length) & ~0x0fff;
2042 if (packet->buf1 != (buf & ~(u64)0x0fff))
2043 packet->cross = 1;
2044
David Brownell53bd6a62006-08-30 14:50:06 -07002045 /* OUT uses multiple start-splits */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002046 if (stream->bEndpointAddress & USB_DIR_IN)
2047 continue;
2048 length = (length + 187) / 188;
2049 if (length > 1) /* BEGIN vs ALL */
2050 length |= 1 << 3;
2051 packet->buf1 |= length;
2052 }
2053}
2054
2055static int
Geyslan G. Bemee906472016-01-25 22:45:14 -03002056sitd_urb_transaction(
Linus Torvalds1da177e2005-04-16 15:20:36 -07002057 struct ehci_iso_stream *stream,
2058 struct ehci_hcd *ehci,
2059 struct urb *urb,
Al Viro55016f12005-10-21 03:21:58 -04002060 gfp_t mem_flags
Linus Torvalds1da177e2005-04-16 15:20:36 -07002061)
2062{
2063 struct ehci_sitd *sitd;
2064 dma_addr_t sitd_dma;
2065 int i;
2066 struct ehci_iso_sched *iso_sched;
2067 unsigned long flags;
2068
Geyslan G. Bemee906472016-01-25 22:45:14 -03002069 iso_sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070 if (iso_sched == NULL)
2071 return -ENOMEM;
2072
Stefan Roese6dbd6822007-05-01 09:29:37 -07002073 sitd_sched_init(ehci, iso_sched, stream, urb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074
2075 /* allocate/init sITDs */
Geyslan G. Bemee906472016-01-25 22:45:14 -03002076 spin_lock_irqsave(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077 for (i = 0; i < urb->number_of_packets; i++) {
2078
2079 /* NOTE: for now, we don't try to handle wraparound cases
2080 * for IN (using sitd->hw_backpointer, like a FSTN), which
2081 * means we never need two sitds for full speed packets.
2082 */
2083
Alan Stern55934eb2012-07-11 11:22:35 -04002084 /*
2085 * Use siTDs from the free list, but not siTDs that may
2086 * still be in use by the hardware.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002087 */
Alan Stern55934eb2012-07-11 11:22:35 -04002088 if (likely(!list_empty(&stream->free_list))) {
2089 sitd = list_first_entry(&stream->free_list,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002090 struct ehci_sitd, sitd_list);
Alan Sternf4289072012-07-11 11:23:07 -04002091 if (sitd->frame == ehci->now_frame)
Alan Stern55934eb2012-07-11 11:22:35 -04002092 goto alloc_sitd;
Geyslan G. Bemee906472016-01-25 22:45:14 -03002093 list_del(&sitd->sitd_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094 sitd_dma = sitd->sitd_dma;
Karsten Wiese3d01f0f2008-02-19 12:31:49 -08002095 } else {
Alan Stern55934eb2012-07-11 11:22:35 -04002096 alloc_sitd:
Geyslan G. Bemee906472016-01-25 22:45:14 -03002097 spin_unlock_irqrestore(&ehci->lock, flags);
2098 sitd = dma_pool_alloc(ehci->sitd_pool, mem_flags,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002099 &sitd_dma);
Geyslan G. Bemee906472016-01-25 22:45:14 -03002100 spin_lock_irqsave(&ehci->lock, flags);
Karsten Wiese3d01f0f2008-02-19 12:31:49 -08002101 if (!sitd) {
2102 iso_sched_free(stream, iso_sched);
2103 spin_unlock_irqrestore(&ehci->lock, flags);
2104 return -ENOMEM;
2105 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106 }
2107
Geyslan G. Bemee2a1d22016-01-25 22:45:18 -03002108 memset(sitd, 0, sizeof(*sitd));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002109 sitd->sitd_dma = sitd_dma;
Alan Stern91a99b52013-10-11 11:28:52 -04002110 sitd->frame = NO_FRAME;
Geyslan G. Bemee906472016-01-25 22:45:14 -03002111 list_add(&sitd->sitd_list, &iso_sched->td_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002112 }
2113
2114 /* temporarily store schedule info in hcpriv */
2115 urb->hcpriv = iso_sched;
2116 urb->error_count = 0;
2117
Geyslan G. Bemee906472016-01-25 22:45:14 -03002118 spin_unlock_irqrestore(&ehci->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119 return 0;
2120}
2121
2122/*-------------------------------------------------------------------------*/
2123
2124static inline void
Stefan Roese6dbd6822007-05-01 09:29:37 -07002125sitd_patch(
2126 struct ehci_hcd *ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127 struct ehci_iso_stream *stream,
2128 struct ehci_sitd *sitd,
2129 struct ehci_iso_sched *iso_sched,
2130 unsigned index
2131)
2132{
Geyslan G. Bemee906472016-01-25 22:45:14 -03002133 struct ehci_iso_packet *uf = &iso_sched->packet[index];
Geyslan G. Bem77d65542016-01-25 22:45:12 -03002134 u64 bufp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002135
Stefan Roese6dbd6822007-05-01 09:29:37 -07002136 sitd->hw_next = EHCI_LIST_END(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002137 sitd->hw_fullspeed_ep = stream->address;
2138 sitd->hw_uframe = stream->splits;
2139 sitd->hw_results = uf->transaction;
Stefan Roese6dbd6822007-05-01 09:29:37 -07002140 sitd->hw_backpointer = EHCI_LIST_END(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002141
2142 bufp = uf->bufp;
Stefan Roese6dbd6822007-05-01 09:29:37 -07002143 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
2144 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145
Stefan Roese6dbd6822007-05-01 09:29:37 -07002146 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002147 if (uf->cross)
2148 bufp += 4096;
Stefan Roese6dbd6822007-05-01 09:29:37 -07002149 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002150 sitd->index = index;
2151}
2152
2153static inline void
Geyslan G. Bemee906472016-01-25 22:45:14 -03002154sitd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002155{
2156 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
Geyslan G. Bemee906472016-01-25 22:45:14 -03002157 sitd->sitd_next = ehci->pshadow[frame];
2158 sitd->hw_next = ehci->periodic[frame];
2159 ehci->pshadow[frame].sitd = sitd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002160 sitd->frame = frame;
Geyslan G. Bemee906472016-01-25 22:45:14 -03002161 wmb();
Stefan Roese6dbd6822007-05-01 09:29:37 -07002162 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002163}
2164
2165/* fit urb's sitds into the selected schedule slot; activate as needed */
Alan Sternb015cb72012-07-11 11:22:10 -04002166static void sitd_link_urb(
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167 struct ehci_hcd *ehci,
2168 struct urb *urb,
2169 unsigned mod,
2170 struct ehci_iso_stream *stream
2171)
2172{
2173 int packet;
2174 unsigned next_uframe;
2175 struct ehci_iso_sched *sched = urb->hcpriv;
2176 struct ehci_sitd *sitd;
2177
2178 next_uframe = stream->next_uframe;
2179
Xenia Ragiadakoufea26ef2013-08-29 11:45:10 +03002180 if (list_empty(&stream->td_list))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002181 /* usbfs ignores TT bandwidth */
2182 ehci_to_hcd(ehci)->self.bandwidth_allocated
2183 += stream->bandwidth;
Alex He05570292010-12-07 10:10:08 +08002184
2185 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
Andiry Xuad935622011-03-01 14:57:05 +08002186 if (ehci->amd_pll_fix == 1)
2187 usb_amd_quirk_pll_disable();
Alex He05570292010-12-07 10:10:08 +08002188 }
2189
Linus Torvalds1da177e2005-04-16 15:20:36 -07002190 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2191
2192 /* fill sITDs frame by frame */
Alan Stern46c73d12013-09-03 13:59:03 -04002193 for (packet = sched->first_packet, sitd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002194 packet < urb->number_of_packets;
2195 packet++) {
2196
2197 /* ASSERT: we have all necessary sitds */
Geyslan G. Bemee906472016-01-25 22:45:14 -03002198 BUG_ON(list_empty(&sched->td_list));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002199
2200 /* ASSERT: no itds for this endpoint in this frame */
2201
Geyslan G. Bemee906472016-01-25 22:45:14 -03002202 sitd = list_entry(sched->td_list.next,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203 struct ehci_sitd, sitd_list);
Geyslan G. Bemee906472016-01-25 22:45:14 -03002204 list_move_tail(&sitd->sitd_list, &stream->td_list);
Alan Stern8c5bf7b2012-07-11 11:22:39 -04002205 sitd->stream = stream;
Karsten Wiese508db8c2009-02-26 01:47:48 +01002206 sitd->urb = urb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002207
Stefan Roese6dbd6822007-05-01 09:29:37 -07002208 sitd_patch(ehci, stream, sitd, sched, packet);
Alan Sternbccbefa2010-07-14 11:03:36 -04002209 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
Linus Torvalds1da177e2005-04-16 15:20:36 -07002210 sitd);
2211
Alan Sternffa02482013-10-11 11:29:03 -04002212 next_uframe += stream->uperiod;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213 }
Alan Sternbccbefa2010-07-14 11:03:36 -04002214 stream->next_uframe = next_uframe & (mod - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215
2216 /* don't need that schedule data any more */
Geyslan G. Bemee906472016-01-25 22:45:14 -03002217 iso_sched_free(stream, sched);
Alan Stern2656a9a2012-11-08 10:17:01 -05002218 urb->hcpriv = stream;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002219
Alan Stern569b3942012-07-11 11:23:00 -04002220 ++ehci->isoc_count;
Alan Sternb015cb72012-07-11 11:22:10 -04002221 enable_periodic(ehci);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002222}
2223
2224/*-------------------------------------------------------------------------*/
2225
2226#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
David Brownell53bd6a62006-08-30 14:50:06 -07002227 | SITD_STS_XACT | SITD_STS_MMF)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002228
David Brownell30bf54e2007-12-16 22:37:40 -08002229/* Process and recycle a completed SITD. Return true iff its urb completed,
2230 * and hence its completion callback probably added things to the hardware
2231 * schedule.
2232 *
2233 * Note that we carefully avoid recycling this descriptor until after any
2234 * completion callback runs, so that it won't be reused quickly. That is,
2235 * assuming (a) no more than two urbs per frame on this endpoint, and also
2236 * (b) only this endpoint's completions submit URBs. It seems some silicon
2237 * corrupts things if you reuse completed descriptors very quickly...
2238 */
Alan Sternf4289072012-07-11 11:23:07 -04002239static bool sitd_complete(struct ehci_hcd *ehci, struct ehci_sitd *sitd)
2240{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002241 struct urb *urb = sitd->urb;
2242 struct usb_iso_packet_descriptor *desc;
2243 u32 t;
Geyslan G. Bem77d65542016-01-25 22:45:12 -03002244 int urb_index;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002245 struct ehci_iso_stream *stream = sitd->stream;
2246 struct usb_device *dev;
Alan Sternf4289072012-07-11 11:23:07 -04002247 bool retval = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002248
2249 urb_index = sitd->index;
Geyslan G. Bemee906472016-01-25 22:45:14 -03002250 desc = &urb->iso_frame_desc[urb_index];
Stefan Roese6dbd6822007-05-01 09:29:37 -07002251 t = hc32_to_cpup(ehci, &sitd->hw_results);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002252
2253 /* report transfer status */
Alan Stern4005ad42012-10-01 10:32:01 -04002254 if (unlikely(t & SITD_ERRS)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255 urb->error_count++;
2256 if (t & SITD_STS_DBE)
Geyslan G. Bemee906472016-01-25 22:45:14 -03002257 desc->status = usb_pipein(urb->pipe)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002258 ? -ENOSR /* hc couldn't read */
2259 : -ECOMM; /* hc couldn't write */
2260 else if (t & SITD_STS_BABBLE)
2261 desc->status = -EOVERFLOW;
2262 else /* XACT, MMF, etc */
2263 desc->status = -EPROTO;
Alan Stern4005ad42012-10-01 10:32:01 -04002264 } else if (unlikely(t & SITD_STS_ACTIVE)) {
2265 /* URB was too late */
2266 urb->error_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002267 } else {
2268 desc->status = 0;
Alan Sternec6d67e2009-06-29 14:34:59 -04002269 desc->actual_length = desc->length - SITD_LENGTH(t);
2270 urb->actual_length += desc->actual_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002271 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002272
2273 /* handle completion now? */
2274 if ((urb_index + 1) != urb->number_of_packets)
David Brownell30bf54e2007-12-16 22:37:40 -08002275 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002276
Geyslan G. Bem1ec27802016-01-25 22:45:16 -03002277 /*
2278 * ASSERT: it's really the last sitd for this urb
2279 * list_for_each_entry (sitd, &stream->td_list, sitd_list)
2280 * BUG_ON(sitd->urb == urb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281 */
2282
David Brownellaa16ca32007-12-30 23:45:19 -08002283 /* give urb back to the driver; completion often (re)submits */
Alan Stern6a8e87b2006-01-19 10:46:27 -05002284 dev = urb->dev;
Alan Stern14c04c02007-08-24 15:40:19 -04002285 ehci_urb_done(ehci, urb, 0);
David Brownell30bf54e2007-12-16 22:37:40 -08002286 retval = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002287 urb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002288
Alan Stern569b3942012-07-11 11:23:00 -04002289 --ehci->isoc_count;
2290 disable_periodic(ehci);
2291
2292 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
Alex He05570292010-12-07 10:10:08 +08002293 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
Andiry Xuad935622011-03-01 14:57:05 +08002294 if (ehci->amd_pll_fix == 1)
2295 usb_amd_quirk_pll_enable();
Alex He05570292010-12-07 10:10:08 +08002296 }
2297
Xenia Ragiadakoufea26ef2013-08-29 11:45:10 +03002298 if (list_is_singular(&stream->td_list))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002299 ehci_to_hcd(ehci)->self.bandwidth_allocated
2300 -= stream->bandwidth;
Alan Stern0e5f2312010-04-08 16:56:37 -04002301
David Brownell30bf54e2007-12-16 22:37:40 -08002302done:
David Brownell30bf54e2007-12-16 22:37:40 -08002303 sitd->urb = NULL;
Alan Stern55934eb2012-07-11 11:22:35 -04002304
2305 /* Add to the end of the free list for later reuse */
2306 list_move_tail(&sitd->sitd_list, &stream->free_list);
2307
2308 /* Recycle the siTDs when the pipeline is empty (ep no longer in use) */
2309 if (list_empty(&stream->td_list)) {
2310 list_splice_tail_init(&stream->free_list,
2311 &ehci->cached_sitd_list);
2312 start_free_itds(ehci);
Alan Stern0e5f2312010-04-08 16:56:37 -04002313 }
Alan Stern55934eb2012-07-11 11:22:35 -04002314
David Brownell30bf54e2007-12-16 22:37:40 -08002315 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002316}
2317
2318
Geyslan G. Bemee906472016-01-25 22:45:14 -03002319static int sitd_submit(struct ehci_hcd *ehci, struct urb *urb,
Al Viro55016f12005-10-21 03:21:58 -04002320 gfp_t mem_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321{
2322 int status = -EINVAL;
2323 unsigned long flags;
2324 struct ehci_iso_stream *stream;
2325
2326 /* Get iso_stream head */
Geyslan G. Bemee906472016-01-25 22:45:14 -03002327 stream = iso_stream_find(ehci, urb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002328 if (stream == NULL) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03002329 ehci_dbg(ehci, "can't get iso stream\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002330 return -ENOMEM;
2331 }
Alan Sternffa02482013-10-11 11:29:03 -04002332 if (urb->interval != stream->ps.period) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03002333 ehci_dbg(ehci, "can't change iso interval %d --> %d\n",
Alan Sternffa02482013-10-11 11:29:03 -04002334 stream->ps.period, urb->interval);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002335 goto done;
2336 }
2337
2338#ifdef EHCI_URB_TRACE
Geyslan G. Bemee906472016-01-25 22:45:14 -03002339 ehci_dbg(ehci,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002340 "submit %p dev%s ep%d%s-iso len %d\n",
2341 urb, urb->dev->devpath,
Geyslan G. Bemee906472016-01-25 22:45:14 -03002342 usb_pipeendpoint(urb->pipe),
2343 usb_pipein(urb->pipe) ? "in" : "out",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002344 urb->transfer_buffer_length);
2345#endif
2346
2347 /* allocate SITDs */
Geyslan G. Bemee906472016-01-25 22:45:14 -03002348 status = sitd_urb_transaction(stream, ehci, urb, mem_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002349 if (status < 0) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03002350 ehci_dbg(ehci, "can't init sitds\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002351 goto done;
2352 }
2353
2354 /* schedule ... need to lock */
Geyslan G. Bemee906472016-01-25 22:45:14 -03002355 spin_lock_irqsave(&ehci->lock, flags);
Alan Stern541c7d42010-06-22 16:39:10 -04002356 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
Benjamin Herrenschmidt8de98402005-11-25 09:59:46 +11002357 status = -ESHUTDOWN;
Alan Sterne9df41c2007-08-08 11:48:02 -04002358 goto done_not_linked;
2359 }
2360 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2361 if (unlikely(status))
2362 goto done_not_linked;
2363 status = iso_stream_schedule(ehci, urb, stream);
Alan Stern46c73d12013-09-03 13:59:03 -04002364 if (likely(status == 0)) {
Geyslan G. Bemee906472016-01-25 22:45:14 -03002365 sitd_link_urb(ehci, urb, ehci->periodic_size << 3, stream);
Alan Stern46c73d12013-09-03 13:59:03 -04002366 } else if (status > 0) {
2367 status = 0;
2368 ehci_urb_done(ehci, urb, 0);
2369 } else {
Alan Sterne9df41c2007-08-08 11:48:02 -04002370 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
Alan Stern46c73d12013-09-03 13:59:03 -04002371 }
Alan Stern8c5bf7b2012-07-11 11:22:39 -04002372 done_not_linked:
Geyslan G. Bemee906472016-01-25 22:45:14 -03002373 spin_unlock_irqrestore(&ehci->lock, flags);
Alan Stern8c5bf7b2012-07-11 11:22:39 -04002374 done:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002375 return status;
2376}
2377
Linus Torvalds1da177e2005-04-16 15:20:36 -07002378/*-------------------------------------------------------------------------*/
2379
Alan Stern569b3942012-07-11 11:23:00 -04002380static void scan_isoc(struct ehci_hcd *ehci)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002381{
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002382 unsigned uf, now_frame, frame;
2383 unsigned fmask = ehci->periodic_size - 1;
2384 bool modified, live;
2385 union ehci_shadow q, *q_p;
2386 __hc32 type, *hw_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002387
2388 /*
2389 * When running, scan from last scan point up to "now"
2390 * else clean up by scanning everything that's left.
2391 * Touches as few pages as possible: cache-friendly.
2392 */
Alan Sternc0c53db2012-07-11 11:21:48 -04002393 if (ehci->rh_state >= EHCI_RH_RUNNING) {
Alan Sternf4289072012-07-11 11:23:07 -04002394 uf = ehci_read_frame_index(ehci);
2395 now_frame = (uf >> 3) & fmask;
2396 live = true;
Karsten Wiese9aa09d22009-02-08 16:07:58 -08002397 } else {
Alan Sternc3ee9b72012-09-28 16:01:23 -04002398 now_frame = (ehci->last_iso_frame - 1) & fmask;
Alan Sternf4289072012-07-11 11:23:07 -04002399 live = false;
Karsten Wiese9aa09d22009-02-08 16:07:58 -08002400 }
Alan Sternf4289072012-07-11 11:23:07 -04002401 ehci->now_frame = now_frame;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002402
Alan Sternb09a61c2013-01-30 16:35:02 -05002403 frame = ehci->last_iso_frame;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404
2405restart:
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002406 /* Scan each element in frame's queue for completions */
2407 q_p = &ehci->pshadow[frame];
2408 hw_p = &ehci->periodic[frame];
2409 q.ptr = q_p->ptr;
2410 type = Q_NEXT_TYPE(ehci, *hw_p);
2411 modified = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002412
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002413 while (q.ptr != NULL) {
2414 switch (hc32_to_cpu(ehci, type)) {
2415 case Q_TYPE_ITD:
2416 /*
2417 * If this ITD is still active, leave it for
2418 * later processing ... check the next entry.
2419 * No need to check for activity unless the
2420 * frame is current.
2421 */
2422 if (frame == now_frame && live) {
2423 rmb();
2424 for (uf = 0; uf < 8; uf++) {
2425 if (q.itd->hw_transaction[uf] &
2426 ITD_ACTIVE(ehci))
Alan Sternb40e43f2008-05-20 16:59:10 -04002427 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002428 }
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002429 if (uf < 8) {
2430 q_p = &q.itd->itd_next;
2431 hw_p = &q.itd->hw_next;
Stefan Roese6dbd6822007-05-01 09:29:37 -07002432 type = Q_NEXT_TYPE(ehci,
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002433 q.itd->hw_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002434 q = *q_p;
2435 break;
2436 }
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002437 }
David Brownell79592b722008-01-07 00:47:42 -08002438
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002439 /*
2440 * Take finished ITDs out of the schedule
2441 * and process them: recycle, maybe report
2442 * URB completion. HC won't cache the
2443 * pointer for much longer, if at all.
2444 */
2445 *q_p = q.itd->itd_next;
2446 if (!ehci->use_dummy_qh ||
2447 q.itd->hw_next != EHCI_LIST_END(ehci))
2448 *hw_p = q.itd->hw_next;
2449 else
2450 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2451 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2452 wmb();
2453 modified = itd_complete(ehci, q.itd);
2454 q = *q_p;
2455 break;
2456 case Q_TYPE_SITD:
2457 /*
2458 * If this SITD is still active, leave it for
2459 * later processing ... check the next entry.
2460 * No need to check for activity unless the
2461 * frame is current.
2462 */
2463 if (((frame == now_frame) ||
2464 (((frame + 1) & fmask) == now_frame))
2465 && live
2466 && (q.sitd->hw_results & SITD_ACTIVE(ehci))) {
2467
2468 q_p = &q.sitd->sitd_next;
2469 hw_p = &q.sitd->hw_next;
Stefan Roese6dbd6822007-05-01 09:29:37 -07002470 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002471 q = *q_p;
2472 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002473 }
2474
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002475 /*
2476 * Take finished SITDs out of the schedule
2477 * and process them: recycle, maybe report
2478 * URB completion.
2479 */
2480 *q_p = q.sitd->sitd_next;
2481 if (!ehci->use_dummy_qh ||
2482 q.sitd->hw_next != EHCI_LIST_END(ehci))
2483 *hw_p = q.sitd->hw_next;
2484 else
2485 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma);
2486 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2487 wmb();
2488 modified = sitd_complete(ehci, q.sitd);
2489 q = *q_p;
2490 break;
2491 default:
2492 ehci_dbg(ehci, "corrupt type %d frame %d shadow %p\n",
2493 type, frame, q.ptr);
2494 /* BUG(); */
2495 /* FALL THROUGH */
2496 case Q_TYPE_QH:
2497 case Q_TYPE_FSTN:
2498 /* End of the iTDs and siTDs */
2499 q.ptr = NULL;
2500 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501 }
2502
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002503 /* Assume completion callbacks modify the queue */
2504 if (unlikely(modified && ehci->isoc_count > 0))
2505 goto restart;
David Brownell53bd6a62006-08-30 14:50:06 -07002506 }
Geyslan G. Bem3f122a92016-01-25 22:45:10 -03002507
2508 /* Stop when we have reached the current frame */
2509 if (frame == now_frame)
2510 return;
2511
2512 /* The last frame may still have active siTDs */
2513 ehci->last_iso_frame = frame;
2514 frame = (frame + 1) & fmask;
2515
2516 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002517}