blob: afd18173cca1286d80e4ead63eab1e7911f91eed [file] [log] [blame]
Kristian Høgsberged568912006-12-19 19:58:35 -05001/* -*- c-basic-offset: 8 -*-
2 *
3 * fw-ohci.c - Driver for OHCI 1394 boards
4 * Copyright (C) 2003-2006 Kristian Hoegsberg <krh@bitplanet.net>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 */
20
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/init.h>
24#include <linux/interrupt.h>
25#include <linux/pci.h>
26#include <linux/delay.h>
27#include <linux/poll.h>
Andrew Mortoncf3e72f2006-12-27 14:36:37 -080028#include <linux/dma-mapping.h>
29
Kristian Høgsberged568912006-12-19 19:58:35 -050030#include <asm/uaccess.h>
31#include <asm/semaphore.h>
32
33#include "fw-transaction.h"
34#include "fw-ohci.h"
35
36#define descriptor_output_more 0
37#define descriptor_output_last (1 << 12)
38#define descriptor_input_more (2 << 12)
39#define descriptor_input_last (3 << 12)
40#define descriptor_status (1 << 11)
41#define descriptor_key_immediate (2 << 8)
42#define descriptor_ping (1 << 7)
43#define descriptor_yy (1 << 6)
44#define descriptor_no_irq (0 << 4)
45#define descriptor_irq_error (1 << 4)
46#define descriptor_irq_always (3 << 4)
47#define descriptor_branch_always (3 << 2)
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -050048#define descriptor_wait (3 << 0)
Kristian Høgsberged568912006-12-19 19:58:35 -050049
50struct descriptor {
51 __le16 req_count;
52 __le16 control;
53 __le32 data_address;
54 __le32 branch_address;
55 __le16 res_count;
56 __le16 transfer_status;
57} __attribute__((aligned(16)));
58
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -050059struct db_descriptor {
60 __le16 first_size;
61 __le16 control;
62 __le16 second_req_count;
63 __le16 first_req_count;
64 __le32 branch_address;
65 __le16 second_res_count;
66 __le16 first_res_count;
67 __le32 reserved0;
68 __le32 first_buffer;
69 __le32 second_buffer;
70 __le32 reserved1;
71} __attribute__((aligned(16)));
72
Kristian Høgsberg72e318e2007-02-06 14:49:31 -050073#define control_set(regs) (regs)
74#define control_clear(regs) ((regs) + 4)
75#define command_ptr(regs) ((regs) + 12)
76#define context_match(regs) ((regs) + 16)
77
Kristian Høgsberg32b46092007-02-06 14:49:30 -050078struct ar_buffer {
79 struct descriptor descriptor;
80 struct ar_buffer *next;
81 __le32 data[0];
82};
83
Kristian Høgsberged568912006-12-19 19:58:35 -050084struct ar_context {
85 struct fw_ohci *ohci;
Kristian Høgsberg32b46092007-02-06 14:49:30 -050086 struct ar_buffer *current_buffer;
87 struct ar_buffer *last_buffer;
88 void *pointer;
Kristian Høgsberg72e318e2007-02-06 14:49:31 -050089 u32 regs;
Kristian Høgsberged568912006-12-19 19:58:35 -050090 struct tasklet_struct tasklet;
91};
92
Kristian Høgsberg30200732007-02-16 17:34:39 -050093struct context;
94
95typedef int (*descriptor_callback_t)(struct context *ctx,
96 struct descriptor *d,
97 struct descriptor *last);
98struct context {
Stefan Richter373b2ed2007-03-04 14:45:18 +010099 struct fw_ohci *ohci;
Kristian Høgsberg30200732007-02-16 17:34:39 -0500100 u32 regs;
Stefan Richter373b2ed2007-03-04 14:45:18 +0100101
Kristian Høgsberg30200732007-02-16 17:34:39 -0500102 struct descriptor *buffer;
103 dma_addr_t buffer_bus;
104 size_t buffer_size;
105 struct descriptor *head_descriptor;
106 struct descriptor *tail_descriptor;
107 struct descriptor *tail_descriptor_last;
108 struct descriptor *prev_descriptor;
109
110 descriptor_callback_t callback;
111
Stefan Richter373b2ed2007-03-04 14:45:18 +0100112 struct tasklet_struct tasklet;
Kristian Høgsberg30200732007-02-16 17:34:39 -0500113};
Kristian Høgsberg30200732007-02-16 17:34:39 -0500114
Kristian Høgsberged568912006-12-19 19:58:35 -0500115#define it_header_sy(v) ((v) << 0)
116#define it_header_tcode(v) ((v) << 4)
117#define it_header_channel(v) ((v) << 8)
118#define it_header_tag(v) ((v) << 14)
119#define it_header_speed(v) ((v) << 16)
120#define it_header_data_length(v) ((v) << 16)
121
122struct iso_context {
123 struct fw_iso_context base;
Kristian Høgsberg30200732007-02-16 17:34:39 -0500124 struct context context;
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -0500125 void *header;
126 size_t header_length;
Kristian Høgsberged568912006-12-19 19:58:35 -0500127};
128
129#define CONFIG_ROM_SIZE 1024
130
131struct fw_ohci {
132 struct fw_card card;
133
Kristian Høgsberge364cf42007-02-16 17:34:49 -0500134 u32 version;
Kristian Høgsberged568912006-12-19 19:58:35 -0500135 __iomem char *registers;
136 dma_addr_t self_id_bus;
137 __le32 *self_id_cpu;
138 struct tasklet_struct bus_reset_tasklet;
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500139 int node_id;
Kristian Høgsberged568912006-12-19 19:58:35 -0500140 int generation;
141 int request_generation;
Kristian Høgsbergd60d7f12007-03-07 12:12:56 -0500142 u32 bus_seconds;
Kristian Høgsberged568912006-12-19 19:58:35 -0500143
144 /* Spinlock for accessing fw_ohci data. Never call out of
145 * this driver with this lock held. */
146 spinlock_t lock;
147 u32 self_id_buffer[512];
148
149 /* Config rom buffers */
150 __be32 *config_rom;
151 dma_addr_t config_rom_bus;
152 __be32 *next_config_rom;
153 dma_addr_t next_config_rom_bus;
154 u32 next_header;
155
156 struct ar_context ar_request_ctx;
157 struct ar_context ar_response_ctx;
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500158 struct context at_request_ctx;
159 struct context at_response_ctx;
Kristian Høgsberged568912006-12-19 19:58:35 -0500160
161 u32 it_context_mask;
162 struct iso_context *it_context_list;
163 u32 ir_context_mask;
164 struct iso_context *ir_context_list;
165};
166
Adrian Bunk95688e92007-01-22 19:17:37 +0100167static inline struct fw_ohci *fw_ohci(struct fw_card *card)
Kristian Høgsberged568912006-12-19 19:58:35 -0500168{
169 return container_of(card, struct fw_ohci, card);
170}
171
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -0500172#define IT_CONTEXT_CYCLE_MATCH_ENABLE 0x80000000
173#define IR_CONTEXT_BUFFER_FILL 0x80000000
174#define IR_CONTEXT_ISOCH_HEADER 0x40000000
175#define IR_CONTEXT_CYCLE_MATCH_ENABLE 0x20000000
176#define IR_CONTEXT_MULTI_CHANNEL_MODE 0x10000000
177#define IR_CONTEXT_DUAL_BUFFER_MODE 0x08000000
Kristian Høgsberged568912006-12-19 19:58:35 -0500178
179#define CONTEXT_RUN 0x8000
180#define CONTEXT_WAKE 0x1000
181#define CONTEXT_DEAD 0x0800
182#define CONTEXT_ACTIVE 0x0400
183
184#define OHCI1394_MAX_AT_REQ_RETRIES 0x2
185#define OHCI1394_MAX_AT_RESP_RETRIES 0x2
186#define OHCI1394_MAX_PHYS_RESP_RETRIES 0x8
187
188#define FW_OHCI_MAJOR 240
189#define OHCI1394_REGISTER_SIZE 0x800
190#define OHCI_LOOP_COUNT 500
191#define OHCI1394_PCI_HCI_Control 0x40
192#define SELF_ID_BUF_SIZE 0x800
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500193#define OHCI_TCODE_PHY_PACKET 0x0e
Kristian Høgsberge364cf42007-02-16 17:34:49 -0500194#define OHCI_VERSION_1_1 0x010010
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500195#define ISO_BUFFER_SIZE (64 * 1024)
196#define AT_BUFFER_SIZE 4096
Kristian Høgsberg0edeefd2007-01-26 00:38:49 -0500197
Kristian Høgsberged568912006-12-19 19:58:35 -0500198static char ohci_driver_name[] = KBUILD_MODNAME;
199
Adrian Bunk95688e92007-01-22 19:17:37 +0100200static inline void reg_write(const struct fw_ohci *ohci, int offset, u32 data)
Kristian Høgsberged568912006-12-19 19:58:35 -0500201{
202 writel(data, ohci->registers + offset);
203}
204
Adrian Bunk95688e92007-01-22 19:17:37 +0100205static inline u32 reg_read(const struct fw_ohci *ohci, int offset)
Kristian Høgsberged568912006-12-19 19:58:35 -0500206{
207 return readl(ohci->registers + offset);
208}
209
Adrian Bunk95688e92007-01-22 19:17:37 +0100210static inline void flush_writes(const struct fw_ohci *ohci)
Kristian Høgsberged568912006-12-19 19:58:35 -0500211{
212 /* Do a dummy read to flush writes. */
213 reg_read(ohci, OHCI1394_Version);
214}
215
216static int
217ohci_update_phy_reg(struct fw_card *card, int addr,
218 int clear_bits, int set_bits)
219{
220 struct fw_ohci *ohci = fw_ohci(card);
221 u32 val, old;
222
223 reg_write(ohci, OHCI1394_PhyControl, OHCI1394_PhyControl_Read(addr));
224 msleep(2);
225 val = reg_read(ohci, OHCI1394_PhyControl);
226 if ((val & OHCI1394_PhyControl_ReadDone) == 0) {
227 fw_error("failed to set phy reg bits.\n");
228 return -EBUSY;
229 }
230
231 old = OHCI1394_PhyControl_ReadData(val);
232 old = (old & ~clear_bits) | set_bits;
233 reg_write(ohci, OHCI1394_PhyControl,
234 OHCI1394_PhyControl_Write(addr, old));
235
236 return 0;
237}
238
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500239static int ar_context_add_page(struct ar_context *ctx)
Kristian Høgsberged568912006-12-19 19:58:35 -0500240{
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500241 struct device *dev = ctx->ohci->card.device;
242 struct ar_buffer *ab;
243 dma_addr_t ab_bus;
244 size_t offset;
245
246 ab = (struct ar_buffer *) __get_free_page(GFP_ATOMIC);
247 if (ab == NULL)
248 return -ENOMEM;
249
250 ab_bus = dma_map_single(dev, ab, PAGE_SIZE, DMA_BIDIRECTIONAL);
251 if (dma_mapping_error(ab_bus)) {
252 free_page((unsigned long) ab);
253 return -ENOMEM;
254 }
255
256 memset(&ab->descriptor, 0, sizeof ab->descriptor);
257 ab->descriptor.control = cpu_to_le16(descriptor_input_more |
258 descriptor_status |
259 descriptor_branch_always);
260 offset = offsetof(struct ar_buffer, data);
261 ab->descriptor.req_count = cpu_to_le16(PAGE_SIZE - offset);
262 ab->descriptor.data_address = cpu_to_le32(ab_bus + offset);
263 ab->descriptor.res_count = cpu_to_le16(PAGE_SIZE - offset);
264 ab->descriptor.branch_address = 0;
265
266 dma_sync_single_for_device(dev, ab_bus, PAGE_SIZE, DMA_BIDIRECTIONAL);
267
268 ctx->last_buffer->descriptor.branch_address = ab_bus | 1;
269 ctx->last_buffer->next = ab;
270 ctx->last_buffer = ab;
271
Kristian Høgsberg72e318e2007-02-06 14:49:31 -0500272 reg_write(ctx->ohci, control_set(ctx->regs), CONTEXT_WAKE);
Kristian Høgsberged568912006-12-19 19:58:35 -0500273 flush_writes(ctx->ohci);
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500274
275 return 0;
Kristian Høgsberged568912006-12-19 19:58:35 -0500276}
277
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500278static __le32 *handle_ar_packet(struct ar_context *ctx, __le32 *buffer)
Kristian Høgsberged568912006-12-19 19:58:35 -0500279{
Kristian Høgsberged568912006-12-19 19:58:35 -0500280 struct fw_ohci *ohci = ctx->ohci;
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500281 struct fw_packet p;
282 u32 status, length, tcode;
Kristian Høgsberg0edeefd2007-01-26 00:38:49 -0500283
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500284 p.header[0] = le32_to_cpu(buffer[0]);
285 p.header[1] = le32_to_cpu(buffer[1]);
286 p.header[2] = le32_to_cpu(buffer[2]);
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500287
288 tcode = (p.header[0] >> 4) & 0x0f;
289 switch (tcode) {
290 case TCODE_WRITE_QUADLET_REQUEST:
291 case TCODE_READ_QUADLET_RESPONSE:
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500292 p.header[3] = (__force __u32) buffer[3];
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500293 p.header_length = 16;
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500294 p.payload_length = 0;
295 break;
296
297 case TCODE_READ_BLOCK_REQUEST :
298 p.header[3] = le32_to_cpu(buffer[3]);
299 p.header_length = 16;
300 p.payload_length = 0;
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500301 break;
302
303 case TCODE_WRITE_BLOCK_REQUEST:
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500304 case TCODE_READ_BLOCK_RESPONSE:
305 case TCODE_LOCK_REQUEST:
306 case TCODE_LOCK_RESPONSE:
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500307 p.header[3] = le32_to_cpu(buffer[3]);
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500308 p.header_length = 16;
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500309 p.payload_length = p.header[3] >> 16;
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500310 break;
311
312 case TCODE_WRITE_RESPONSE:
313 case TCODE_READ_QUADLET_REQUEST:
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500314 case OHCI_TCODE_PHY_PACKET:
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500315 p.header_length = 12;
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500316 p.payload_length = 0;
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500317 break;
318 }
319
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500320 p.payload = (void *) buffer + p.header_length;
321
322 /* FIXME: What to do about evt_* errors? */
323 length = (p.header_length + p.payload_length + 3) / 4;
324 status = le32_to_cpu(buffer[length]);
325
326 p.ack = ((status >> 16) & 0x1f) - 16;
327 p.speed = (status >> 21) & 0x7;
328 p.timestamp = status & 0xffff;
329 p.generation = ohci->request_generation;
Kristian Høgsberged568912006-12-19 19:58:35 -0500330
331 /* The OHCI bus reset handler synthesizes a phy packet with
332 * the new generation number when a bus reset happens (see
333 * section 8.4.2.3). This helps us determine when a request
334 * was received and make sure we send the response in the same
335 * generation. We only need this for requests; for responses
336 * we use the unique tlabel for finding the matching
337 * request. */
338
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500339 if (p.ack + 16 == 0x09)
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500340 ohci->request_generation = (buffer[2] >> 16) & 0xff;
Kristian Høgsberged568912006-12-19 19:58:35 -0500341 else if (ctx == &ohci->ar_request_ctx)
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500342 fw_core_handle_request(&ohci->card, &p);
Kristian Høgsberged568912006-12-19 19:58:35 -0500343 else
Kristian Høgsberg2639a6f2007-01-26 00:37:57 -0500344 fw_core_handle_response(&ohci->card, &p);
Kristian Høgsberged568912006-12-19 19:58:35 -0500345
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500346 return buffer + length + 1;
347}
Kristian Høgsberged568912006-12-19 19:58:35 -0500348
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500349static void ar_context_tasklet(unsigned long data)
350{
351 struct ar_context *ctx = (struct ar_context *)data;
352 struct fw_ohci *ohci = ctx->ohci;
353 struct ar_buffer *ab;
354 struct descriptor *d;
355 void *buffer, *end;
Kristian Høgsberged568912006-12-19 19:58:35 -0500356
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500357 ab = ctx->current_buffer;
358 d = &ab->descriptor;
Kristian Høgsberged568912006-12-19 19:58:35 -0500359
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500360 if (d->res_count == 0) {
361 size_t size, rest, offset;
362
363 /* This descriptor is finished and we may have a
364 * packet split across this and the next buffer. We
365 * reuse the page for reassembling the split packet. */
366
367 offset = offsetof(struct ar_buffer, data);
368 dma_unmap_single(ohci->card.device,
369 ab->descriptor.data_address - offset,
370 PAGE_SIZE, DMA_BIDIRECTIONAL);
371
372 buffer = ab;
373 ab = ab->next;
374 d = &ab->descriptor;
375 size = buffer + PAGE_SIZE - ctx->pointer;
376 rest = le16_to_cpu(d->req_count) - le16_to_cpu(d->res_count);
377 memmove(buffer, ctx->pointer, size);
378 memcpy(buffer + size, ab->data, rest);
379 ctx->current_buffer = ab;
380 ctx->pointer = (void *) ab->data + rest;
381 end = buffer + size + rest;
382
383 while (buffer < end)
384 buffer = handle_ar_packet(ctx, buffer);
385
386 free_page((unsigned long)buffer);
387 ar_context_add_page(ctx);
388 } else {
389 buffer = ctx->pointer;
390 ctx->pointer = end =
391 (void *) ab + PAGE_SIZE - le16_to_cpu(d->res_count);
392
393 while (buffer < end)
394 buffer = handle_ar_packet(ctx, buffer);
395 }
Kristian Høgsberged568912006-12-19 19:58:35 -0500396}
397
398static int
Kristian Høgsberg72e318e2007-02-06 14:49:31 -0500399ar_context_init(struct ar_context *ctx, struct fw_ohci *ohci, u32 regs)
Kristian Høgsberged568912006-12-19 19:58:35 -0500400{
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500401 struct ar_buffer ab;
Kristian Høgsberged568912006-12-19 19:58:35 -0500402
Kristian Høgsberg72e318e2007-02-06 14:49:31 -0500403 ctx->regs = regs;
404 ctx->ohci = ohci;
405 ctx->last_buffer = &ab;
Kristian Høgsberged568912006-12-19 19:58:35 -0500406 tasklet_init(&ctx->tasklet, ar_context_tasklet, (unsigned long)ctx);
407
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500408 ar_context_add_page(ctx);
409 ar_context_add_page(ctx);
410 ctx->current_buffer = ab.next;
411 ctx->pointer = ctx->current_buffer->data;
412
Kristian Høgsberg72e318e2007-02-06 14:49:31 -0500413 reg_write(ctx->ohci, command_ptr(ctx->regs), ab.descriptor.branch_address);
414 reg_write(ctx->ohci, control_set(ctx->regs), CONTEXT_RUN);
Kristian Høgsberg32b46092007-02-06 14:49:30 -0500415 flush_writes(ctx->ohci);
Kristian Høgsberged568912006-12-19 19:58:35 -0500416
417 return 0;
418}
Stefan Richter373b2ed2007-03-04 14:45:18 +0100419
Kristian Høgsberg30200732007-02-16 17:34:39 -0500420static void context_tasklet(unsigned long data)
421{
422 struct context *ctx = (struct context *) data;
423 struct fw_ohci *ohci = ctx->ohci;
424 struct descriptor *d, *last;
425 u32 address;
426 int z;
427
428 dma_sync_single_for_cpu(ohci->card.device, ctx->buffer_bus,
429 ctx->buffer_size, DMA_TO_DEVICE);
430
431 d = ctx->tail_descriptor;
432 last = ctx->tail_descriptor_last;
433
434 while (last->branch_address != 0) {
435 address = le32_to_cpu(last->branch_address);
436 z = address & 0xf;
437 d = ctx->buffer + (address - ctx->buffer_bus) / sizeof *d;
438 last = (z == 2) ? d : d + z - 1;
439
440 if (!ctx->callback(ctx, d, last))
441 break;
442
443 ctx->tail_descriptor = d;
444 ctx->tail_descriptor_last = last;
445 }
446}
447
448static int
449context_init(struct context *ctx, struct fw_ohci *ohci,
450 size_t buffer_size, u32 regs,
451 descriptor_callback_t callback)
452{
453 ctx->ohci = ohci;
454 ctx->regs = regs;
455 ctx->buffer_size = buffer_size;
456 ctx->buffer = kmalloc(buffer_size, GFP_KERNEL);
457 if (ctx->buffer == NULL)
458 return -ENOMEM;
459
460 tasklet_init(&ctx->tasklet, context_tasklet, (unsigned long)ctx);
461 ctx->callback = callback;
462
463 ctx->buffer_bus =
464 dma_map_single(ohci->card.device, ctx->buffer,
465 buffer_size, DMA_TO_DEVICE);
466 if (dma_mapping_error(ctx->buffer_bus)) {
467 kfree(ctx->buffer);
468 return -ENOMEM;
469 }
470
471 ctx->head_descriptor = ctx->buffer;
472 ctx->prev_descriptor = ctx->buffer;
473 ctx->tail_descriptor = ctx->buffer;
474 ctx->tail_descriptor_last = ctx->buffer;
475
476 /* We put a dummy descriptor in the buffer that has a NULL
477 * branch address and looks like it's been sent. That way we
478 * have a descriptor to append DMA programs to. Also, the
479 * ring buffer invariant is that it always has at least one
480 * element so that head == tail means buffer full. */
481
482 memset(ctx->head_descriptor, 0, sizeof *ctx->head_descriptor);
483 ctx->head_descriptor->control = cpu_to_le16(descriptor_output_last);
484 ctx->head_descriptor->transfer_status = cpu_to_le16(0x8011);
485 ctx->head_descriptor++;
486
487 return 0;
488}
489
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -0500490static void
Kristian Høgsberg30200732007-02-16 17:34:39 -0500491context_release(struct context *ctx)
492{
493 struct fw_card *card = &ctx->ohci->card;
494
495 dma_unmap_single(card->device, ctx->buffer_bus,
496 ctx->buffer_size, DMA_TO_DEVICE);
497 kfree(ctx->buffer);
498}
499
500static struct descriptor *
501context_get_descriptors(struct context *ctx, int z, dma_addr_t *d_bus)
502{
503 struct descriptor *d, *tail, *end;
504
505 d = ctx->head_descriptor;
506 tail = ctx->tail_descriptor;
507 end = ctx->buffer + ctx->buffer_size / sizeof(struct descriptor);
508
509 if (d + z <= tail) {
510 goto has_space;
511 } else if (d > tail && d + z <= end) {
512 goto has_space;
513 } else if (d > tail && ctx->buffer + z <= tail) {
514 d = ctx->buffer;
515 goto has_space;
516 }
517
518 return NULL;
519
520 has_space:
521 memset(d, 0, z * sizeof *d);
522 *d_bus = ctx->buffer_bus + (d - ctx->buffer) * sizeof *d;
523
524 return d;
525}
526
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -0500527static void context_run(struct context *ctx, u32 extra)
Kristian Høgsberg30200732007-02-16 17:34:39 -0500528{
529 struct fw_ohci *ohci = ctx->ohci;
530
531 reg_write(ohci, command_ptr(ctx->regs),
532 le32_to_cpu(ctx->tail_descriptor_last->branch_address));
533 reg_write(ohci, control_clear(ctx->regs), ~0);
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -0500534 reg_write(ohci, control_set(ctx->regs), CONTEXT_RUN | extra);
Kristian Høgsberg30200732007-02-16 17:34:39 -0500535 flush_writes(ohci);
536}
537
538static void context_append(struct context *ctx,
539 struct descriptor *d, int z, int extra)
540{
541 dma_addr_t d_bus;
542
543 d_bus = ctx->buffer_bus + (d - ctx->buffer) * sizeof *d;
544
545 ctx->head_descriptor = d + z + extra;
546 ctx->prev_descriptor->branch_address = cpu_to_le32(d_bus | z);
547 ctx->prev_descriptor = z == 2 ? d : d + z - 1;
548
549 dma_sync_single_for_device(ctx->ohci->card.device, ctx->buffer_bus,
550 ctx->buffer_size, DMA_TO_DEVICE);
551
552 reg_write(ctx->ohci, control_set(ctx->regs), CONTEXT_WAKE);
553 flush_writes(ctx->ohci);
554}
555
556static void context_stop(struct context *ctx)
557{
558 u32 reg;
Kristian Høgsbergb8295662007-02-16 17:34:42 -0500559 int i;
Kristian Høgsberg30200732007-02-16 17:34:39 -0500560
561 reg_write(ctx->ohci, control_clear(ctx->regs), CONTEXT_RUN);
Kristian Høgsbergb8295662007-02-16 17:34:42 -0500562 flush_writes(ctx->ohci);
Kristian Høgsberg30200732007-02-16 17:34:39 -0500563
Kristian Høgsbergb8295662007-02-16 17:34:42 -0500564 for (i = 0; i < 10; i++) {
565 reg = reg_read(ctx->ohci, control_set(ctx->regs));
566 if ((reg & CONTEXT_ACTIVE) == 0)
567 break;
568
569 fw_notify("context_stop: still active (0x%08x)\n", reg);
570 msleep(1);
571 }
Kristian Høgsberg30200732007-02-16 17:34:39 -0500572}
Kristian Høgsberged568912006-12-19 19:58:35 -0500573
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500574struct driver_data {
Kristian Høgsberged568912006-12-19 19:58:35 -0500575 struct fw_packet *packet;
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500576};
577
578/* This function apppends a packet to the DMA queue for transmission.
579 * Must always be called with the ochi->lock held to ensure proper
580 * generation handling and locking around packet queue manipulation. */
581static int
582at_context_queue_packet(struct context *ctx, struct fw_packet *packet)
583{
Kristian Høgsberged568912006-12-19 19:58:35 -0500584 struct fw_ohci *ohci = ctx->ohci;
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500585 dma_addr_t d_bus, payload_bus;
586 struct driver_data *driver_data;
587 struct descriptor *d, *last;
588 __le32 *header;
Kristian Høgsberged568912006-12-19 19:58:35 -0500589 int z, tcode;
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500590 u32 reg;
Kristian Høgsberged568912006-12-19 19:58:35 -0500591
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500592 d = context_get_descriptors(ctx, 4, &d_bus);
593 if (d == NULL) {
594 packet->ack = RCODE_SEND_ERROR;
595 return -1;
Kristian Høgsberged568912006-12-19 19:58:35 -0500596 }
597
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500598 d[0].control = cpu_to_le16(descriptor_key_immediate);
599 d[0].res_count = cpu_to_le16(packet->timestamp);
600
Kristian Høgsberged568912006-12-19 19:58:35 -0500601 /* The DMA format for asyncronous link packets is different
602 * from the IEEE1394 layout, so shift the fields around
603 * accordingly. If header_length is 8, it's a PHY packet, to
604 * which we need to prepend an extra quadlet. */
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500605
606 header = (__le32 *) &d[1];
Kristian Høgsberged568912006-12-19 19:58:35 -0500607 if (packet->header_length > 8) {
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500608 header[0] = cpu_to_le32((packet->header[0] & 0xffff) |
609 (packet->speed << 16));
610 header[1] = cpu_to_le32((packet->header[1] & 0xffff) |
611 (packet->header[0] & 0xffff0000));
612 header[2] = cpu_to_le32(packet->header[2]);
Kristian Høgsberged568912006-12-19 19:58:35 -0500613
614 tcode = (packet->header[0] >> 4) & 0x0f;
615 if (TCODE_IS_BLOCK_PACKET(tcode))
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500616 header[3] = cpu_to_le32(packet->header[3]);
Kristian Høgsberged568912006-12-19 19:58:35 -0500617 else
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500618 header[3] = (__force __le32) packet->header[3];
619
620 d[0].req_count = cpu_to_le16(packet->header_length);
Kristian Høgsberged568912006-12-19 19:58:35 -0500621 } else {
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500622 header[0] = cpu_to_le32((OHCI1394_phy_tcode << 4) |
623 (packet->speed << 16));
624 header[1] = cpu_to_le32(packet->header[0]);
625 header[2] = cpu_to_le32(packet->header[1]);
626 d[0].req_count = cpu_to_le16(12);
Kristian Høgsberged568912006-12-19 19:58:35 -0500627 }
628
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500629 driver_data = (struct driver_data *) &d[3];
630 driver_data->packet = packet;
Kristian Høgsberg20d11672007-03-26 19:18:19 -0400631 packet->driver_data = driver_data;
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500632
633 if (packet->payload_length > 0) {
634 payload_bus =
635 dma_map_single(ohci->card.device, packet->payload,
636 packet->payload_length, DMA_TO_DEVICE);
637 if (dma_mapping_error(payload_bus)) {
638 packet->ack = RCODE_SEND_ERROR;
639 return -1;
640 }
641
642 d[2].req_count = cpu_to_le16(packet->payload_length);
643 d[2].data_address = cpu_to_le32(payload_bus);
644 last = &d[2];
645 z = 3;
646 } else {
647 last = &d[0];
648 z = 2;
649 }
650
651 last->control |= cpu_to_le16(descriptor_output_last |
652 descriptor_irq_always |
653 descriptor_branch_always);
654
Kristian Høgsberged568912006-12-19 19:58:35 -0500655 /* FIXME: Document how the locking works. */
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500656 if (ohci->generation != packet->generation) {
657 packet->ack = RCODE_GENERATION;
658 return -1;
Kristian Høgsberged568912006-12-19 19:58:35 -0500659 }
Kristian Høgsberged568912006-12-19 19:58:35 -0500660
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500661 context_append(ctx, d, z, 4 - z);
Kristian Høgsberged568912006-12-19 19:58:35 -0500662
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500663 /* If the context isn't already running, start it up. */
Kristian Høgsberg72e318e2007-02-06 14:49:31 -0500664 reg = reg_read(ctx->ohci, control_set(ctx->regs));
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500665 if ((reg & CONTEXT_ACTIVE) == 0)
666 context_run(ctx, 0);
Kristian Høgsberged568912006-12-19 19:58:35 -0500667
668 return 0;
669}
670
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500671static int handle_at_packet(struct context *context,
672 struct descriptor *d,
673 struct descriptor *last)
674{
675 struct driver_data *driver_data;
676 struct fw_packet *packet;
677 struct fw_ohci *ohci = context->ohci;
678 dma_addr_t payload_bus;
679 int evt;
680
681 if (last->transfer_status == 0)
682 /* This descriptor isn't done yet, stop iteration. */
683 return 0;
684
685 driver_data = (struct driver_data *) &d[3];
686 packet = driver_data->packet;
687 if (packet == NULL)
688 /* This packet was cancelled, just continue. */
689 return 1;
690
691 payload_bus = le32_to_cpu(last->data_address);
692 if (payload_bus != 0)
693 dma_unmap_single(ohci->card.device, payload_bus,
694 packet->payload_length, DMA_TO_DEVICE);
695
696 evt = le16_to_cpu(last->transfer_status) & 0x1f;
697 packet->timestamp = le16_to_cpu(last->res_count);
698
699 switch (evt) {
700 case OHCI1394_evt_timeout:
701 /* Async response transmit timed out. */
702 packet->ack = RCODE_CANCELLED;
703 break;
704
705 case OHCI1394_evt_flushed:
706 /* The packet was flushed should give same error as
707 * when we try to use a stale generation count. */
708 packet->ack = RCODE_GENERATION;
709 break;
710
711 case OHCI1394_evt_missing_ack:
712 /* Using a valid (current) generation count, but the
713 * node is not on the bus or not sending acks. */
714 packet->ack = RCODE_NO_ACK;
715 break;
716
717 case ACK_COMPLETE + 0x10:
718 case ACK_PENDING + 0x10:
719 case ACK_BUSY_X + 0x10:
720 case ACK_BUSY_A + 0x10:
721 case ACK_BUSY_B + 0x10:
722 case ACK_DATA_ERROR + 0x10:
723 case ACK_TYPE_ERROR + 0x10:
724 packet->ack = evt - 0x10;
725 break;
726
727 default:
728 packet->ack = RCODE_SEND_ERROR;
729 break;
730 }
731
732 packet->callback(packet, &ohci->card, packet->ack);
733
734 return 1;
735}
736
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500737#define header_get_destination(q) (((q) >> 16) & 0xffff)
Kristian Høgsberg93c4cce2007-01-26 00:38:26 -0500738#define header_get_tcode(q) (((q) >> 4) & 0x0f)
739#define header_get_offset_high(q) (((q) >> 0) & 0xffff)
740#define header_get_data_length(q) (((q) >> 16) & 0xffff)
741#define header_get_extended_tcode(q) (((q) >> 0) & 0xffff)
742
743static void
744handle_local_rom(struct fw_ohci *ohci, struct fw_packet *packet, u32 csr)
745{
746 struct fw_packet response;
747 int tcode, length, i;
748
749 tcode = header_get_tcode(packet->header[0]);
750 if (TCODE_IS_BLOCK_PACKET(tcode))
751 length = header_get_data_length(packet->header[3]);
752 else
753 length = 4;
754
755 i = csr - CSR_CONFIG_ROM;
756 if (i + length > CONFIG_ROM_SIZE) {
757 fw_fill_response(&response, packet->header,
758 RCODE_ADDRESS_ERROR, NULL, 0);
759 } else if (!TCODE_IS_READ_REQUEST(tcode)) {
760 fw_fill_response(&response, packet->header,
761 RCODE_TYPE_ERROR, NULL, 0);
762 } else {
763 fw_fill_response(&response, packet->header, RCODE_COMPLETE,
764 (void *) ohci->config_rom + i, length);
765 }
766
767 fw_core_handle_response(&ohci->card, &response);
768}
769
770static void
771handle_local_lock(struct fw_ohci *ohci, struct fw_packet *packet, u32 csr)
772{
773 struct fw_packet response;
774 int tcode, length, ext_tcode, sel;
775 __be32 *payload, lock_old;
776 u32 lock_arg, lock_data;
777
778 tcode = header_get_tcode(packet->header[0]);
779 length = header_get_data_length(packet->header[3]);
780 payload = packet->payload;
781 ext_tcode = header_get_extended_tcode(packet->header[3]);
782
783 if (tcode == TCODE_LOCK_REQUEST &&
784 ext_tcode == EXTCODE_COMPARE_SWAP && length == 8) {
785 lock_arg = be32_to_cpu(payload[0]);
786 lock_data = be32_to_cpu(payload[1]);
787 } else if (tcode == TCODE_READ_QUADLET_REQUEST) {
788 lock_arg = 0;
789 lock_data = 0;
790 } else {
791 fw_fill_response(&response, packet->header,
792 RCODE_TYPE_ERROR, NULL, 0);
793 goto out;
794 }
795
796 sel = (csr - CSR_BUS_MANAGER_ID) / 4;
797 reg_write(ohci, OHCI1394_CSRData, lock_data);
798 reg_write(ohci, OHCI1394_CSRCompareData, lock_arg);
799 reg_write(ohci, OHCI1394_CSRControl, sel);
800
801 if (reg_read(ohci, OHCI1394_CSRControl) & 0x80000000)
802 lock_old = cpu_to_be32(reg_read(ohci, OHCI1394_CSRData));
803 else
804 fw_notify("swap not done yet\n");
805
806 fw_fill_response(&response, packet->header,
807 RCODE_COMPLETE, &lock_old, sizeof lock_old);
808 out:
809 fw_core_handle_response(&ohci->card, &response);
810}
811
812static void
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500813handle_local_request(struct context *ctx, struct fw_packet *packet)
Kristian Høgsberg93c4cce2007-01-26 00:38:26 -0500814{
815 u64 offset;
816 u32 csr;
817
Kristian Høgsberg473d28c2007-03-07 12:12:55 -0500818 if (ctx == &ctx->ohci->at_request_ctx) {
819 packet->ack = ACK_PENDING;
820 packet->callback(packet, &ctx->ohci->card, packet->ack);
821 }
Kristian Høgsberg93c4cce2007-01-26 00:38:26 -0500822
823 offset =
824 ((unsigned long long)
825 header_get_offset_high(packet->header[1]) << 32) |
826 packet->header[2];
827 csr = offset - CSR_REGISTER_BASE;
828
829 /* Handle config rom reads. */
830 if (csr >= CSR_CONFIG_ROM && csr < CSR_CONFIG_ROM_END)
831 handle_local_rom(ctx->ohci, packet, csr);
832 else switch (csr) {
833 case CSR_BUS_MANAGER_ID:
834 case CSR_BANDWIDTH_AVAILABLE:
835 case CSR_CHANNELS_AVAILABLE_HI:
836 case CSR_CHANNELS_AVAILABLE_LO:
837 handle_local_lock(ctx->ohci, packet, csr);
838 break;
839 default:
840 if (ctx == &ctx->ohci->at_request_ctx)
841 fw_core_handle_request(&ctx->ohci->card, packet);
842 else
843 fw_core_handle_response(&ctx->ohci->card, packet);
844 break;
845 }
Kristian Høgsberg473d28c2007-03-07 12:12:55 -0500846
847 if (ctx == &ctx->ohci->at_response_ctx) {
848 packet->ack = ACK_COMPLETE;
849 packet->callback(packet, &ctx->ohci->card, packet->ack);
850 }
Kristian Høgsberg93c4cce2007-01-26 00:38:26 -0500851}
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500852
Kristian Høgsberged568912006-12-19 19:58:35 -0500853static void
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500854at_context_transmit(struct context *ctx, struct fw_packet *packet)
Kristian Høgsberged568912006-12-19 19:58:35 -0500855{
Kristian Høgsberged568912006-12-19 19:58:35 -0500856 unsigned long flags;
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500857 int retval;
Kristian Høgsberged568912006-12-19 19:58:35 -0500858
859 spin_lock_irqsave(&ctx->ohci->lock, flags);
860
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500861 if (header_get_destination(packet->header[0]) == ctx->ohci->node_id &&
862 ctx->ohci->generation == packet->generation) {
Kristian Høgsberg93c4cce2007-01-26 00:38:26 -0500863 spin_unlock_irqrestore(&ctx->ohci->lock, flags);
864 handle_local_request(ctx, packet);
865 return;
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500866 }
Kristian Høgsberged568912006-12-19 19:58:35 -0500867
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500868 retval = at_context_queue_packet(ctx, packet);
Kristian Høgsberged568912006-12-19 19:58:35 -0500869 spin_unlock_irqrestore(&ctx->ohci->lock, flags);
870
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500871 if (retval < 0)
872 packet->callback(packet, &ctx->ohci->card, packet->ack);
873
Kristian Høgsberged568912006-12-19 19:58:35 -0500874}
875
876static void bus_reset_tasklet(unsigned long data)
877{
878 struct fw_ohci *ohci = (struct fw_ohci *)data;
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500879 int self_id_count, i, j, reg;
Kristian Høgsberged568912006-12-19 19:58:35 -0500880 int generation, new_generation;
881 unsigned long flags;
882
883 reg = reg_read(ohci, OHCI1394_NodeID);
884 if (!(reg & OHCI1394_NodeID_idValid)) {
885 fw_error("node ID not valid, new bus reset in progress\n");
886 return;
887 }
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500888 ohci->node_id = reg & 0xffff;
Kristian Høgsberged568912006-12-19 19:58:35 -0500889
890 /* The count in the SelfIDCount register is the number of
891 * bytes in the self ID receive buffer. Since we also receive
892 * the inverted quadlets and a header quadlet, we shift one
893 * bit extra to get the actual number of self IDs. */
894
895 self_id_count = (reg_read(ohci, OHCI1394_SelfIDCount) >> 3) & 0x3ff;
896 generation = (le32_to_cpu(ohci->self_id_cpu[0]) >> 16) & 0xff;
897
898 for (i = 1, j = 0; j < self_id_count; i += 2, j++) {
899 if (ohci->self_id_cpu[i] != ~ohci->self_id_cpu[i + 1])
900 fw_error("inconsistent self IDs\n");
901 ohci->self_id_buffer[j] = le32_to_cpu(ohci->self_id_cpu[i]);
902 }
903
904 /* Check the consistency of the self IDs we just read. The
905 * problem we face is that a new bus reset can start while we
906 * read out the self IDs from the DMA buffer. If this happens,
907 * the DMA buffer will be overwritten with new self IDs and we
908 * will read out inconsistent data. The OHCI specification
909 * (section 11.2) recommends a technique similar to
910 * linux/seqlock.h, where we remember the generation of the
911 * self IDs in the buffer before reading them out and compare
912 * it to the current generation after reading them out. If
913 * the two generations match we know we have a consistent set
914 * of self IDs. */
915
916 new_generation = (reg_read(ohci, OHCI1394_SelfIDCount) >> 16) & 0xff;
917 if (new_generation != generation) {
918 fw_notify("recursive bus reset detected, "
919 "discarding self ids\n");
920 return;
921 }
922
923 /* FIXME: Document how the locking works. */
924 spin_lock_irqsave(&ohci->lock, flags);
925
926 ohci->generation = generation;
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -0500927 context_stop(&ohci->at_request_ctx);
928 context_stop(&ohci->at_response_ctx);
Kristian Høgsberged568912006-12-19 19:58:35 -0500929 reg_write(ohci, OHCI1394_IntEventClear, OHCI1394_busReset);
930
931 /* This next bit is unrelated to the AT context stuff but we
932 * have to do it under the spinlock also. If a new config rom
933 * was set up before this reset, the old one is now no longer
934 * in use and we can free it. Update the config rom pointers
935 * to point to the current config rom and clear the
936 * next_config_rom pointer so a new udpate can take place. */
937
938 if (ohci->next_config_rom != NULL) {
939 dma_free_coherent(ohci->card.device, CONFIG_ROM_SIZE,
940 ohci->config_rom, ohci->config_rom_bus);
941 ohci->config_rom = ohci->next_config_rom;
942 ohci->config_rom_bus = ohci->next_config_rom_bus;
943 ohci->next_config_rom = NULL;
944
945 /* Restore config_rom image and manually update
946 * config_rom registers. Writing the header quadlet
947 * will indicate that the config rom is ready, so we
948 * do that last. */
949 reg_write(ohci, OHCI1394_BusOptions,
950 be32_to_cpu(ohci->config_rom[2]));
951 ohci->config_rom[0] = cpu_to_be32(ohci->next_header);
952 reg_write(ohci, OHCI1394_ConfigROMhdr, ohci->next_header);
953 }
954
955 spin_unlock_irqrestore(&ohci->lock, flags);
956
Kristian Høgsberge636fe22007-01-26 00:38:04 -0500957 fw_core_handle_bus_reset(&ohci->card, ohci->node_id, generation,
Kristian Høgsberged568912006-12-19 19:58:35 -0500958 self_id_count, ohci->self_id_buffer);
959}
960
961static irqreturn_t irq_handler(int irq, void *data)
962{
963 struct fw_ohci *ohci = data;
Kristian Høgsbergd60d7f12007-03-07 12:12:56 -0500964 u32 event, iso_event, cycle_time;
Kristian Høgsberged568912006-12-19 19:58:35 -0500965 int i;
966
967 event = reg_read(ohci, OHCI1394_IntEventClear);
968
969 if (!event)
970 return IRQ_NONE;
971
972 reg_write(ohci, OHCI1394_IntEventClear, event);
973
974 if (event & OHCI1394_selfIDComplete)
975 tasklet_schedule(&ohci->bus_reset_tasklet);
976
977 if (event & OHCI1394_RQPkt)
978 tasklet_schedule(&ohci->ar_request_ctx.tasklet);
979
980 if (event & OHCI1394_RSPkt)
981 tasklet_schedule(&ohci->ar_response_ctx.tasklet);
982
983 if (event & OHCI1394_reqTxComplete)
984 tasklet_schedule(&ohci->at_request_ctx.tasklet);
985
986 if (event & OHCI1394_respTxComplete)
987 tasklet_schedule(&ohci->at_response_ctx.tasklet);
988
Kristian Høgsbergc8894752007-02-16 17:34:36 -0500989 iso_event = reg_read(ohci, OHCI1394_IsoRecvIntEventClear);
Kristian Høgsberged568912006-12-19 19:58:35 -0500990 reg_write(ohci, OHCI1394_IsoRecvIntEventClear, iso_event);
991
992 while (iso_event) {
993 i = ffs(iso_event) - 1;
Kristian Høgsberg30200732007-02-16 17:34:39 -0500994 tasklet_schedule(&ohci->ir_context_list[i].context.tasklet);
Kristian Høgsberged568912006-12-19 19:58:35 -0500995 iso_event &= ~(1 << i);
996 }
997
Kristian Høgsbergc8894752007-02-16 17:34:36 -0500998 iso_event = reg_read(ohci, OHCI1394_IsoXmitIntEventClear);
Kristian Høgsberged568912006-12-19 19:58:35 -0500999 reg_write(ohci, OHCI1394_IsoXmitIntEventClear, iso_event);
1000
1001 while (iso_event) {
1002 i = ffs(iso_event) - 1;
Kristian Høgsberg30200732007-02-16 17:34:39 -05001003 tasklet_schedule(&ohci->it_context_list[i].context.tasklet);
Kristian Høgsberged568912006-12-19 19:58:35 -05001004 iso_event &= ~(1 << i);
1005 }
1006
Kristian Høgsbergd60d7f12007-03-07 12:12:56 -05001007 if (event & OHCI1394_cycle64Seconds) {
1008 cycle_time = reg_read(ohci, OHCI1394_IsochronousCycleTimer);
1009 if ((cycle_time & 0x80000000) == 0)
1010 ohci->bus_seconds++;
1011 }
1012
Kristian Høgsberged568912006-12-19 19:58:35 -05001013 return IRQ_HANDLED;
1014}
1015
1016static int ohci_enable(struct fw_card *card, u32 *config_rom, size_t length)
1017{
1018 struct fw_ohci *ohci = fw_ohci(card);
1019 struct pci_dev *dev = to_pci_dev(card->device);
1020
1021 /* When the link is not yet enabled, the atomic config rom
1022 * update mechanism described below in ohci_set_config_rom()
1023 * is not active. We have to update ConfigRomHeader and
1024 * BusOptions manually, and the write to ConfigROMmap takes
1025 * effect immediately. We tie this to the enabling of the
1026 * link, so we have a valid config rom before enabling - the
1027 * OHCI requires that ConfigROMhdr and BusOptions have valid
1028 * values before enabling.
1029 *
1030 * However, when the ConfigROMmap is written, some controllers
1031 * always read back quadlets 0 and 2 from the config rom to
1032 * the ConfigRomHeader and BusOptions registers on bus reset.
1033 * They shouldn't do that in this initial case where the link
1034 * isn't enabled. This means we have to use the same
1035 * workaround here, setting the bus header to 0 and then write
1036 * the right values in the bus reset tasklet.
1037 */
1038
1039 ohci->next_config_rom =
1040 dma_alloc_coherent(ohci->card.device, CONFIG_ROM_SIZE,
1041 &ohci->next_config_rom_bus, GFP_KERNEL);
1042 if (ohci->next_config_rom == NULL)
1043 return -ENOMEM;
1044
1045 memset(ohci->next_config_rom, 0, CONFIG_ROM_SIZE);
1046 fw_memcpy_to_be32(ohci->next_config_rom, config_rom, length * 4);
1047
1048 ohci->next_header = config_rom[0];
1049 ohci->next_config_rom[0] = 0;
1050 reg_write(ohci, OHCI1394_ConfigROMhdr, 0);
1051 reg_write(ohci, OHCI1394_BusOptions, config_rom[2]);
1052 reg_write(ohci, OHCI1394_ConfigROMmap, ohci->next_config_rom_bus);
1053
1054 reg_write(ohci, OHCI1394_AsReqFilterHiSet, 0x80000000);
1055
1056 if (request_irq(dev->irq, irq_handler,
Thomas Gleixner65efffa2007-03-05 18:19:51 -08001057 IRQF_SHARED, ohci_driver_name, ohci)) {
Kristian Høgsberged568912006-12-19 19:58:35 -05001058 fw_error("Failed to allocate shared interrupt %d.\n",
1059 dev->irq);
1060 dma_free_coherent(ohci->card.device, CONFIG_ROM_SIZE,
1061 ohci->config_rom, ohci->config_rom_bus);
1062 return -EIO;
1063 }
1064
1065 reg_write(ohci, OHCI1394_HCControlSet,
1066 OHCI1394_HCControl_linkEnable |
1067 OHCI1394_HCControl_BIBimageValid);
1068 flush_writes(ohci);
1069
1070 /* We are ready to go, initiate bus reset to finish the
1071 * initialization. */
1072
1073 fw_core_initiate_bus_reset(&ohci->card, 1);
1074
1075 return 0;
1076}
1077
1078static int
1079ohci_set_config_rom(struct fw_card *card, u32 *config_rom, size_t length)
1080{
1081 struct fw_ohci *ohci;
1082 unsigned long flags;
1083 int retval = 0;
1084 __be32 *next_config_rom;
1085 dma_addr_t next_config_rom_bus;
1086
1087 ohci = fw_ohci(card);
1088
1089 /* When the OHCI controller is enabled, the config rom update
1090 * mechanism is a bit tricky, but easy enough to use. See
1091 * section 5.5.6 in the OHCI specification.
1092 *
1093 * The OHCI controller caches the new config rom address in a
1094 * shadow register (ConfigROMmapNext) and needs a bus reset
1095 * for the changes to take place. When the bus reset is
1096 * detected, the controller loads the new values for the
1097 * ConfigRomHeader and BusOptions registers from the specified
1098 * config rom and loads ConfigROMmap from the ConfigROMmapNext
1099 * shadow register. All automatically and atomically.
1100 *
1101 * Now, there's a twist to this story. The automatic load of
1102 * ConfigRomHeader and BusOptions doesn't honor the
1103 * noByteSwapData bit, so with a be32 config rom, the
1104 * controller will load be32 values in to these registers
1105 * during the atomic update, even on litte endian
1106 * architectures. The workaround we use is to put a 0 in the
1107 * header quadlet; 0 is endian agnostic and means that the
1108 * config rom isn't ready yet. In the bus reset tasklet we
1109 * then set up the real values for the two registers.
1110 *
1111 * We use ohci->lock to avoid racing with the code that sets
1112 * ohci->next_config_rom to NULL (see bus_reset_tasklet).
1113 */
1114
1115 next_config_rom =
1116 dma_alloc_coherent(ohci->card.device, CONFIG_ROM_SIZE,
1117 &next_config_rom_bus, GFP_KERNEL);
1118 if (next_config_rom == NULL)
1119 return -ENOMEM;
1120
1121 spin_lock_irqsave(&ohci->lock, flags);
1122
1123 if (ohci->next_config_rom == NULL) {
1124 ohci->next_config_rom = next_config_rom;
1125 ohci->next_config_rom_bus = next_config_rom_bus;
1126
1127 memset(ohci->next_config_rom, 0, CONFIG_ROM_SIZE);
1128 fw_memcpy_to_be32(ohci->next_config_rom, config_rom,
1129 length * 4);
1130
1131 ohci->next_header = config_rom[0];
1132 ohci->next_config_rom[0] = 0;
1133
1134 reg_write(ohci, OHCI1394_ConfigROMmap,
1135 ohci->next_config_rom_bus);
1136 } else {
1137 dma_free_coherent(ohci->card.device, CONFIG_ROM_SIZE,
1138 next_config_rom, next_config_rom_bus);
1139 retval = -EBUSY;
1140 }
1141
1142 spin_unlock_irqrestore(&ohci->lock, flags);
1143
1144 /* Now initiate a bus reset to have the changes take
1145 * effect. We clean up the old config rom memory and DMA
1146 * mappings in the bus reset tasklet, since the OHCI
1147 * controller could need to access it before the bus reset
1148 * takes effect. */
1149 if (retval == 0)
1150 fw_core_initiate_bus_reset(&ohci->card, 1);
1151
1152 return retval;
1153}
1154
1155static void ohci_send_request(struct fw_card *card, struct fw_packet *packet)
1156{
1157 struct fw_ohci *ohci = fw_ohci(card);
1158
1159 at_context_transmit(&ohci->at_request_ctx, packet);
1160}
1161
1162static void ohci_send_response(struct fw_card *card, struct fw_packet *packet)
1163{
1164 struct fw_ohci *ohci = fw_ohci(card);
1165
1166 at_context_transmit(&ohci->at_response_ctx, packet);
1167}
1168
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001169static int ohci_cancel_packet(struct fw_card *card, struct fw_packet *packet)
1170{
1171 struct fw_ohci *ohci = fw_ohci(card);
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001172 struct context *ctx = &ohci->at_request_ctx;
1173 struct driver_data *driver_data = packet->driver_data;
1174 int retval = -ENOENT;
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001175
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001176 tasklet_disable(&ctx->tasklet);
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001177
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001178 if (packet->ack != 0)
1179 goto out;
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001180
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001181 driver_data->packet = NULL;
1182 packet->ack = RCODE_CANCELLED;
1183 packet->callback(packet, &ohci->card, packet->ack);
1184 retval = 0;
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001185
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001186 out:
1187 tasklet_enable(&ctx->tasklet);
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001188
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001189 return retval;
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001190}
1191
Kristian Høgsberged568912006-12-19 19:58:35 -05001192static int
1193ohci_enable_phys_dma(struct fw_card *card, int node_id, int generation)
1194{
1195 struct fw_ohci *ohci = fw_ohci(card);
1196 unsigned long flags;
Stefan Richter907293d2007-01-23 21:11:43 +01001197 int n, retval = 0;
Kristian Høgsberged568912006-12-19 19:58:35 -05001198
Stefan Richter907293d2007-01-23 21:11:43 +01001199 /* FIXME: Make sure this bitmask is cleared when we clear the busReset
1200 * interrupt bit. Clear physReqResourceAllBuses on bus reset. */
Kristian Høgsberged568912006-12-19 19:58:35 -05001201
1202 spin_lock_irqsave(&ohci->lock, flags);
1203
1204 if (ohci->generation != generation) {
1205 retval = -ESTALE;
1206 goto out;
1207 }
1208
Stefan Richter907293d2007-01-23 21:11:43 +01001209 /* NOTE, if the node ID contains a non-local bus ID, physical DMA is
1210 * enabled for _all_ nodes on remote buses. */
1211
1212 n = (node_id & 0xffc0) == LOCAL_BUS ? node_id & 0x3f : 63;
1213 if (n < 32)
1214 reg_write(ohci, OHCI1394_PhyReqFilterLoSet, 1 << n);
1215 else
1216 reg_write(ohci, OHCI1394_PhyReqFilterHiSet, 1 << (n - 32));
1217
Kristian Høgsberged568912006-12-19 19:58:35 -05001218 flush_writes(ohci);
Kristian Høgsberged568912006-12-19 19:58:35 -05001219 out:
Stefan Richter6cad95f2007-01-21 20:46:45 +01001220 spin_unlock_irqrestore(&ohci->lock, flags);
Kristian Høgsberged568912006-12-19 19:58:35 -05001221 return retval;
1222}
Stefan Richter373b2ed2007-03-04 14:45:18 +01001223
Kristian Høgsbergd60d7f12007-03-07 12:12:56 -05001224static u64
1225ohci_get_bus_time(struct fw_card *card)
1226{
1227 struct fw_ohci *ohci = fw_ohci(card);
1228 u32 cycle_time;
1229 u64 bus_time;
1230
1231 cycle_time = reg_read(ohci, OHCI1394_IsochronousCycleTimer);
1232 bus_time = ((u64) ohci->bus_seconds << 32) | cycle_time;
1233
1234 return bus_time;
1235}
1236
Kristian Høgsbergd2746dc2007-02-16 17:34:46 -05001237static int handle_ir_dualbuffer_packet(struct context *context,
1238 struct descriptor *d,
1239 struct descriptor *last)
Kristian Høgsberged568912006-12-19 19:58:35 -05001240{
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001241 struct iso_context *ctx =
1242 container_of(context, struct iso_context, context);
1243 struct db_descriptor *db = (struct db_descriptor *) d;
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001244 __le32 *ir_header;
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001245 size_t header_length;
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001246 void *p, *end;
1247 int i;
Kristian Høgsbergd2746dc2007-02-16 17:34:46 -05001248
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001249 if (db->first_res_count > 0 && db->second_res_count > 0)
1250 /* This descriptor isn't done yet, stop iteration. */
1251 return 0;
Kristian Høgsberged568912006-12-19 19:58:35 -05001252
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001253 header_length = le16_to_cpu(db->first_req_count) -
1254 le16_to_cpu(db->first_res_count);
1255
1256 i = ctx->header_length;
1257 p = db + 1;
1258 end = p + header_length;
1259 while (p < end && i + ctx->base.header_size <= PAGE_SIZE) {
1260 memcpy(ctx->header + i, p + 4, ctx->base.header_size);
1261 i += ctx->base.header_size;
1262 p += ctx->base.header_size + 4;
1263 }
1264
1265 ctx->header_length = i;
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001266
1267 if (le16_to_cpu(db->control) & descriptor_irq_always) {
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001268 ir_header = (__le32 *) (db + 1);
1269 ctx->base.callback(&ctx->base,
1270 le32_to_cpu(ir_header[0]) & 0xffff,
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001271 ctx->header_length, ctx->header,
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001272 ctx->base.callback_data);
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001273 ctx->header_length = 0;
1274 }
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001275
1276 return 1;
Kristian Høgsberged568912006-12-19 19:58:35 -05001277}
1278
Kristian Høgsberg30200732007-02-16 17:34:39 -05001279static int handle_it_packet(struct context *context,
1280 struct descriptor *d,
1281 struct descriptor *last)
Kristian Høgsberged568912006-12-19 19:58:35 -05001282{
Kristian Høgsberg30200732007-02-16 17:34:39 -05001283 struct iso_context *ctx =
1284 container_of(context, struct iso_context, context);
Stefan Richter373b2ed2007-03-04 14:45:18 +01001285
Kristian Høgsberg30200732007-02-16 17:34:39 -05001286 if (last->transfer_status == 0)
1287 /* This descriptor isn't done yet, stop iteration. */
1288 return 0;
Kristian Høgsberged568912006-12-19 19:58:35 -05001289
Kristian Høgsberg30200732007-02-16 17:34:39 -05001290 if (le16_to_cpu(last->control) & descriptor_irq_always)
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001291 ctx->base.callback(&ctx->base, le16_to_cpu(last->res_count),
1292 0, NULL, ctx->base.callback_data);
Kristian Høgsberged568912006-12-19 19:58:35 -05001293
Kristian Høgsberg30200732007-02-16 17:34:39 -05001294 return 1;
Kristian Høgsberged568912006-12-19 19:58:35 -05001295}
1296
Kristian Høgsberg30200732007-02-16 17:34:39 -05001297static struct fw_iso_context *
Kristian Høgsbergeb0306e2007-03-14 17:34:54 -04001298ohci_allocate_iso_context(struct fw_card *card, int type, size_t header_size)
Kristian Høgsberged568912006-12-19 19:58:35 -05001299{
1300 struct fw_ohci *ohci = fw_ohci(card);
1301 struct iso_context *ctx, *list;
Kristian Høgsberg30200732007-02-16 17:34:39 -05001302 descriptor_callback_t callback;
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001303 u32 *mask, regs;
Kristian Høgsberged568912006-12-19 19:58:35 -05001304 unsigned long flags;
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001305 int index, retval = -ENOMEM;
Kristian Høgsberged568912006-12-19 19:58:35 -05001306
1307 if (type == FW_ISO_CONTEXT_TRANSMIT) {
1308 mask = &ohci->it_context_mask;
1309 list = ohci->it_context_list;
Kristian Høgsberg30200732007-02-16 17:34:39 -05001310 callback = handle_it_packet;
Kristian Høgsberged568912006-12-19 19:58:35 -05001311 } else {
Stefan Richter373b2ed2007-03-04 14:45:18 +01001312 mask = &ohci->ir_context_mask;
1313 list = ohci->ir_context_list;
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001314 callback = handle_ir_dualbuffer_packet;
Kristian Høgsberged568912006-12-19 19:58:35 -05001315 }
1316
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001317 /* FIXME: We need a fallback for pre 1.1 OHCI. */
Kristian Høgsberge364cf42007-02-16 17:34:49 -05001318 if (callback == handle_ir_dualbuffer_packet &&
1319 ohci->version < OHCI_VERSION_1_1)
1320 return ERR_PTR(-EINVAL);
1321
Kristian Høgsberged568912006-12-19 19:58:35 -05001322 spin_lock_irqsave(&ohci->lock, flags);
1323 index = ffs(*mask) - 1;
1324 if (index >= 0)
1325 *mask &= ~(1 << index);
1326 spin_unlock_irqrestore(&ohci->lock, flags);
1327
1328 if (index < 0)
1329 return ERR_PTR(-EBUSY);
1330
Stefan Richter373b2ed2007-03-04 14:45:18 +01001331 if (type == FW_ISO_CONTEXT_TRANSMIT)
1332 regs = OHCI1394_IsoXmitContextBase(index);
1333 else
1334 regs = OHCI1394_IsoRcvContextBase(index);
1335
Kristian Høgsberged568912006-12-19 19:58:35 -05001336 ctx = &list[index];
1337 memset(ctx, 0, sizeof *ctx);
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001338 ctx->header_length = 0;
1339 ctx->header = (void *) __get_free_page(GFP_KERNEL);
1340 if (ctx->header == NULL)
1341 goto out;
1342
Kristian Høgsberg30200732007-02-16 17:34:39 -05001343 retval = context_init(&ctx->context, ohci, ISO_BUFFER_SIZE,
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001344 regs, callback);
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001345 if (retval < 0)
1346 goto out_with_header;
Kristian Høgsberged568912006-12-19 19:58:35 -05001347
1348 return &ctx->base;
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001349
1350 out_with_header:
1351 free_page((unsigned long)ctx->header);
1352 out:
1353 spin_lock_irqsave(&ohci->lock, flags);
1354 *mask |= 1 << index;
1355 spin_unlock_irqrestore(&ohci->lock, flags);
1356
1357 return ERR_PTR(retval);
Kristian Høgsberged568912006-12-19 19:58:35 -05001358}
1359
Kristian Høgsbergeb0306e2007-03-14 17:34:54 -04001360static int ohci_start_iso(struct fw_iso_context *base,
1361 s32 cycle, u32 sync, u32 tags)
Kristian Høgsberged568912006-12-19 19:58:35 -05001362{
Stefan Richter373b2ed2007-03-04 14:45:18 +01001363 struct iso_context *ctx = container_of(base, struct iso_context, base);
Kristian Høgsberg30200732007-02-16 17:34:39 -05001364 struct fw_ohci *ohci = ctx->context.ohci;
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001365 u32 cycle_match = 0;
Kristian Høgsberged568912006-12-19 19:58:35 -05001366 int index;
1367
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001368 if (ctx->base.type == FW_ISO_CONTEXT_TRANSMIT) {
1369 index = ctx - ohci->it_context_list;
1370 if (cycle > 0)
1371 cycle_match = IT_CONTEXT_CYCLE_MATCH_ENABLE |
1372 (cycle & 0x7fff) << 16;
Kristian Høgsberg21efb3c2007-02-16 17:34:50 -05001373
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001374 reg_write(ohci, OHCI1394_IsoXmitIntEventClear, 1 << index);
1375 reg_write(ohci, OHCI1394_IsoXmitIntMaskSet, 1 << index);
1376 context_run(&ctx->context, cycle_match);
1377 } else {
1378 index = ctx - ohci->ir_context_list;
Kristian Høgsberged568912006-12-19 19:58:35 -05001379
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001380 reg_write(ohci, OHCI1394_IsoRecvIntEventClear, 1 << index);
1381 reg_write(ohci, OHCI1394_IsoRecvIntMaskSet, 1 << index);
1382 reg_write(ohci, context_match(ctx->context.regs),
Kristian Høgsbergeb0306e2007-03-14 17:34:54 -04001383 (tags << 28) | (sync << 8) | ctx->base.channel);
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001384 context_run(&ctx->context,
1385 IR_CONTEXT_DUAL_BUFFER_MODE |
1386 IR_CONTEXT_ISOCH_HEADER);
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001387 }
Kristian Høgsberged568912006-12-19 19:58:35 -05001388
1389 return 0;
1390}
1391
Kristian Høgsbergb8295662007-02-16 17:34:42 -05001392static int ohci_stop_iso(struct fw_iso_context *base)
1393{
1394 struct fw_ohci *ohci = fw_ohci(base->card);
Stefan Richter373b2ed2007-03-04 14:45:18 +01001395 struct iso_context *ctx = container_of(base, struct iso_context, base);
Kristian Høgsbergb8295662007-02-16 17:34:42 -05001396 int index;
1397
1398 if (ctx->base.type == FW_ISO_CONTEXT_TRANSMIT) {
1399 index = ctx - ohci->it_context_list;
1400 reg_write(ohci, OHCI1394_IsoXmitIntMaskClear, 1 << index);
1401 } else {
1402 index = ctx - ohci->ir_context_list;
1403 reg_write(ohci, OHCI1394_IsoRecvIntMaskClear, 1 << index);
1404 }
1405 flush_writes(ohci);
1406 context_stop(&ctx->context);
1407
1408 return 0;
1409}
1410
Kristian Høgsberged568912006-12-19 19:58:35 -05001411static void ohci_free_iso_context(struct fw_iso_context *base)
1412{
1413 struct fw_ohci *ohci = fw_ohci(base->card);
Stefan Richter373b2ed2007-03-04 14:45:18 +01001414 struct iso_context *ctx = container_of(base, struct iso_context, base);
Kristian Høgsberged568912006-12-19 19:58:35 -05001415 unsigned long flags;
1416 int index;
1417
Kristian Høgsbergb8295662007-02-16 17:34:42 -05001418 ohci_stop_iso(base);
1419 context_release(&ctx->context);
Kristian Høgsberg9b32d5f2007-02-16 17:34:44 -05001420 free_page((unsigned long)ctx->header);
Kristian Høgsbergb8295662007-02-16 17:34:42 -05001421
Kristian Høgsberged568912006-12-19 19:58:35 -05001422 spin_lock_irqsave(&ohci->lock, flags);
1423
1424 if (ctx->base.type == FW_ISO_CONTEXT_TRANSMIT) {
1425 index = ctx - ohci->it_context_list;
Kristian Høgsberged568912006-12-19 19:58:35 -05001426 ohci->it_context_mask |= 1 << index;
1427 } else {
1428 index = ctx - ohci->ir_context_list;
Kristian Høgsberged568912006-12-19 19:58:35 -05001429 ohci->ir_context_mask |= 1 << index;
1430 }
Kristian Høgsberged568912006-12-19 19:58:35 -05001431
1432 spin_unlock_irqrestore(&ohci->lock, flags);
1433}
1434
1435static int
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001436ohci_queue_iso_transmit(struct fw_iso_context *base,
1437 struct fw_iso_packet *packet,
1438 struct fw_iso_buffer *buffer,
1439 unsigned long payload)
Kristian Høgsberged568912006-12-19 19:58:35 -05001440{
Stefan Richter373b2ed2007-03-04 14:45:18 +01001441 struct iso_context *ctx = container_of(base, struct iso_context, base);
Kristian Høgsberg30200732007-02-16 17:34:39 -05001442 struct descriptor *d, *last, *pd;
Kristian Høgsberged568912006-12-19 19:58:35 -05001443 struct fw_iso_packet *p;
1444 __le32 *header;
Kristian Høgsberg9aad8122007-02-16 17:34:38 -05001445 dma_addr_t d_bus, page_bus;
Kristian Høgsberged568912006-12-19 19:58:35 -05001446 u32 z, header_z, payload_z, irq;
1447 u32 payload_index, payload_end_index, next_page_index;
Kristian Høgsberg30200732007-02-16 17:34:39 -05001448 int page, end_page, i, length, offset;
Kristian Høgsberged568912006-12-19 19:58:35 -05001449
1450 /* FIXME: Cycle lost behavior should be configurable: lose
1451 * packet, retransmit or terminate.. */
1452
1453 p = packet;
Kristian Høgsberg9aad8122007-02-16 17:34:38 -05001454 payload_index = payload;
Kristian Høgsberged568912006-12-19 19:58:35 -05001455
1456 if (p->skip)
1457 z = 1;
1458 else
1459 z = 2;
1460 if (p->header_length > 0)
1461 z++;
1462
1463 /* Determine the first page the payload isn't contained in. */
1464 end_page = PAGE_ALIGN(payload_index + p->payload_length) >> PAGE_SHIFT;
1465 if (p->payload_length > 0)
1466 payload_z = end_page - (payload_index >> PAGE_SHIFT);
1467 else
1468 payload_z = 0;
1469
1470 z += payload_z;
1471
1472 /* Get header size in number of descriptors. */
1473 header_z = DIV_ROUND_UP(p->header_length, sizeof *d);
1474
Kristian Høgsberg30200732007-02-16 17:34:39 -05001475 d = context_get_descriptors(&ctx->context, z + header_z, &d_bus);
1476 if (d == NULL)
1477 return -ENOMEM;
Kristian Høgsberged568912006-12-19 19:58:35 -05001478
1479 if (!p->skip) {
1480 d[0].control = cpu_to_le16(descriptor_key_immediate);
1481 d[0].req_count = cpu_to_le16(8);
1482
1483 header = (__le32 *) &d[1];
1484 header[0] = cpu_to_le32(it_header_sy(p->sy) |
1485 it_header_tag(p->tag) |
1486 it_header_tcode(TCODE_STREAM_DATA) |
1487 it_header_channel(ctx->base.channel) |
1488 it_header_speed(ctx->base.speed));
1489 header[1] =
1490 cpu_to_le32(it_header_data_length(p->header_length +
1491 p->payload_length));
1492 }
1493
1494 if (p->header_length > 0) {
1495 d[2].req_count = cpu_to_le16(p->header_length);
1496 d[2].data_address = cpu_to_le32(d_bus + z * sizeof *d);
1497 memcpy(&d[z], p->header, p->header_length);
1498 }
1499
1500 pd = d + z - payload_z;
1501 payload_end_index = payload_index + p->payload_length;
1502 for (i = 0; i < payload_z; i++) {
1503 page = payload_index >> PAGE_SHIFT;
1504 offset = payload_index & ~PAGE_MASK;
1505 next_page_index = (page + 1) << PAGE_SHIFT;
1506 length =
1507 min(next_page_index, payload_end_index) - payload_index;
1508 pd[i].req_count = cpu_to_le16(length);
Kristian Høgsberg9aad8122007-02-16 17:34:38 -05001509
1510 page_bus = page_private(buffer->pages[page]);
1511 pd[i].data_address = cpu_to_le32(page_bus + offset);
Kristian Høgsberged568912006-12-19 19:58:35 -05001512
1513 payload_index += length;
1514 }
1515
Kristian Høgsberged568912006-12-19 19:58:35 -05001516 if (p->interrupt)
1517 irq = descriptor_irq_always;
1518 else
1519 irq = descriptor_no_irq;
1520
Kristian Høgsberg30200732007-02-16 17:34:39 -05001521 last = z == 2 ? d : d + z - 1;
Kristian Høgsbergcbb59da2007-02-16 17:34:35 -05001522 last->control |= cpu_to_le16(descriptor_output_last |
1523 descriptor_status |
1524 descriptor_branch_always |
1525 irq);
Kristian Høgsberged568912006-12-19 19:58:35 -05001526
Kristian Høgsberg30200732007-02-16 17:34:39 -05001527 context_append(&ctx->context, d, z, header_z);
Kristian Høgsberged568912006-12-19 19:58:35 -05001528
1529 return 0;
1530}
Stefan Richter373b2ed2007-03-04 14:45:18 +01001531
Kristian Høgsberg98b6cbe2007-02-16 17:34:51 -05001532static int
Kristian Høgsbergd2746dc2007-02-16 17:34:46 -05001533ohci_queue_iso_receive_dualbuffer(struct fw_iso_context *base,
1534 struct fw_iso_packet *packet,
1535 struct fw_iso_buffer *buffer,
1536 unsigned long payload)
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001537{
1538 struct iso_context *ctx = container_of(base, struct iso_context, base);
1539 struct db_descriptor *db = NULL;
1540 struct descriptor *d;
1541 struct fw_iso_packet *p;
1542 dma_addr_t d_bus, page_bus;
1543 u32 z, header_z, length, rest;
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001544 int page, offset, packet_count, header_size;
Stefan Richter373b2ed2007-03-04 14:45:18 +01001545
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001546 /* FIXME: Cycle lost behavior should be configurable: lose
1547 * packet, retransmit or terminate.. */
1548
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001549 if (packet->skip) {
1550 d = context_get_descriptors(&ctx->context, 2, &d_bus);
1551 if (d == NULL)
1552 return -ENOMEM;
1553
1554 db = (struct db_descriptor *) d;
1555 db->control = cpu_to_le16(descriptor_status |
1556 descriptor_branch_always |
1557 descriptor_wait);
1558 db->first_size = cpu_to_le16(ctx->base.header_size + 4);
1559 context_append(&ctx->context, d, 2, 0);
1560 }
Kristian Høgsberg98b6cbe2007-02-16 17:34:51 -05001561
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001562 p = packet;
1563 z = 2;
1564
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001565 /* The OHCI controller puts the status word in the header
1566 * buffer too, so we need 4 extra bytes per packet. */
1567 packet_count = p->header_length / ctx->base.header_size;
1568 header_size = packet_count * (ctx->base.header_size + 4);
1569
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001570 /* Get header size in number of descriptors. */
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001571 header_z = DIV_ROUND_UP(header_size, sizeof *d);
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001572 page = payload >> PAGE_SHIFT;
1573 offset = payload & ~PAGE_MASK;
1574 rest = p->payload_length;
1575
1576 /* FIXME: OHCI 1.0 doesn't support dual buffer receive */
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001577 /* FIXME: make packet-per-buffer/dual-buffer a context option */
1578 while (rest > 0) {
1579 d = context_get_descriptors(&ctx->context,
1580 z + header_z, &d_bus);
1581 if (d == NULL)
1582 return -ENOMEM;
1583
1584 db = (struct db_descriptor *) d;
1585 db->control = cpu_to_le16(descriptor_status |
1586 descriptor_branch_always);
Kristian Høgsbergc70dc782007-03-14 17:34:53 -04001587 db->first_size = cpu_to_le16(ctx->base.header_size + 4);
1588 db->first_req_count = cpu_to_le16(header_size);
Kristian Høgsberg1e1d1962007-02-16 17:34:45 -05001589 db->first_res_count = db->first_req_count;
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001590 db->first_buffer = cpu_to_le32(d_bus + sizeof *db);
Stefan Richter373b2ed2007-03-04 14:45:18 +01001591
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001592 if (offset + rest < PAGE_SIZE)
1593 length = rest;
1594 else
1595 length = PAGE_SIZE - offset;
1596
Kristian Høgsberg1e1d1962007-02-16 17:34:45 -05001597 db->second_req_count = cpu_to_le16(length);
1598 db->second_res_count = db->second_req_count;
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001599 page_bus = page_private(buffer->pages[page]);
1600 db->second_buffer = cpu_to_le32(page_bus + offset);
1601
Kristian Høgsbergcb2d2cd2007-02-16 17:34:47 -05001602 if (p->interrupt && length == rest)
1603 db->control |= cpu_to_le16(descriptor_irq_always);
1604
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001605 context_append(&ctx->context, d, z, header_z);
1606 offset = (offset + length) & ~PAGE_MASK;
1607 rest -= length;
1608 page++;
1609 }
1610
Kristian Høgsbergd2746dc2007-02-16 17:34:46 -05001611 return 0;
1612}
Kristian Høgsberg21efb3c2007-02-16 17:34:50 -05001613
Kristian Høgsbergd2746dc2007-02-16 17:34:46 -05001614static int
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001615ohci_queue_iso(struct fw_iso_context *base,
1616 struct fw_iso_packet *packet,
1617 struct fw_iso_buffer *buffer,
1618 unsigned long payload)
1619{
Kristian Høgsberge364cf42007-02-16 17:34:49 -05001620 struct iso_context *ctx = container_of(base, struct iso_context, base);
1621
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001622 if (base->type == FW_ISO_CONTEXT_TRANSMIT)
1623 return ohci_queue_iso_transmit(base, packet, buffer, payload);
Kristian Høgsberge364cf42007-02-16 17:34:49 -05001624 else if (ctx->context.ohci->version >= OHCI_VERSION_1_1)
Kristian Høgsbergd2746dc2007-02-16 17:34:46 -05001625 return ohci_queue_iso_receive_dualbuffer(base, packet,
1626 buffer, payload);
Kristian Høgsberge364cf42007-02-16 17:34:49 -05001627 else
1628 /* FIXME: Implement fallback for OHCI 1.0 controllers. */
1629 return -EINVAL;
Kristian Høgsberg295e3fe2007-02-16 17:34:40 -05001630}
1631
Stefan Richter21ebcd12007-01-14 15:29:07 +01001632static const struct fw_card_driver ohci_driver = {
Kristian Høgsberged568912006-12-19 19:58:35 -05001633 .name = ohci_driver_name,
1634 .enable = ohci_enable,
1635 .update_phy_reg = ohci_update_phy_reg,
1636 .set_config_rom = ohci_set_config_rom,
1637 .send_request = ohci_send_request,
1638 .send_response = ohci_send_response,
Kristian Høgsberg730c32f2007-02-06 14:49:32 -05001639 .cancel_packet = ohci_cancel_packet,
Kristian Høgsberged568912006-12-19 19:58:35 -05001640 .enable_phys_dma = ohci_enable_phys_dma,
Kristian Høgsbergd60d7f12007-03-07 12:12:56 -05001641 .get_bus_time = ohci_get_bus_time,
Kristian Høgsberged568912006-12-19 19:58:35 -05001642
1643 .allocate_iso_context = ohci_allocate_iso_context,
1644 .free_iso_context = ohci_free_iso_context,
1645 .queue_iso = ohci_queue_iso,
Kristian Høgsberg69cdb722007-02-16 17:34:41 -05001646 .start_iso = ohci_start_iso,
Kristian Høgsbergb8295662007-02-16 17:34:42 -05001647 .stop_iso = ohci_stop_iso,
Kristian Høgsberged568912006-12-19 19:58:35 -05001648};
1649
1650static int software_reset(struct fw_ohci *ohci)
1651{
1652 int i;
1653
1654 reg_write(ohci, OHCI1394_HCControlSet, OHCI1394_HCControl_softReset);
1655
1656 for (i = 0; i < OHCI_LOOP_COUNT; i++) {
1657 if ((reg_read(ohci, OHCI1394_HCControlSet) &
1658 OHCI1394_HCControl_softReset) == 0)
1659 return 0;
1660 msleep(1);
1661 }
1662
1663 return -EBUSY;
1664}
1665
1666/* ---------- pci subsystem interface ---------- */
1667
1668enum {
1669 CLEANUP_SELF_ID,
1670 CLEANUP_REGISTERS,
1671 CLEANUP_IOMEM,
1672 CLEANUP_DISABLE,
1673 CLEANUP_PUT_CARD,
1674};
1675
1676static int cleanup(struct fw_ohci *ohci, int stage, int code)
1677{
1678 struct pci_dev *dev = to_pci_dev(ohci->card.device);
1679
1680 switch (stage) {
1681 case CLEANUP_SELF_ID:
1682 dma_free_coherent(ohci->card.device, SELF_ID_BUF_SIZE,
1683 ohci->self_id_cpu, ohci->self_id_bus);
1684 case CLEANUP_REGISTERS:
1685 kfree(ohci->it_context_list);
1686 kfree(ohci->ir_context_list);
1687 pci_iounmap(dev, ohci->registers);
1688 case CLEANUP_IOMEM:
1689 pci_release_region(dev, 0);
1690 case CLEANUP_DISABLE:
1691 pci_disable_device(dev);
1692 case CLEANUP_PUT_CARD:
1693 fw_card_put(&ohci->card);
1694 }
1695
1696 return code;
1697}
1698
1699static int __devinit
1700pci_probe(struct pci_dev *dev, const struct pci_device_id *ent)
1701{
1702 struct fw_ohci *ohci;
Kristian Høgsberge364cf42007-02-16 17:34:49 -05001703 u32 bus_options, max_receive, link_speed;
Kristian Høgsberged568912006-12-19 19:58:35 -05001704 u64 guid;
1705 int error_code;
1706 size_t size;
1707
1708 ohci = kzalloc(sizeof *ohci, GFP_KERNEL);
1709 if (ohci == NULL) {
1710 fw_error("Could not malloc fw_ohci data.\n");
1711 return -ENOMEM;
1712 }
1713
1714 fw_card_initialize(&ohci->card, &ohci_driver, &dev->dev);
1715
1716 if (pci_enable_device(dev)) {
1717 fw_error("Failed to enable OHCI hardware.\n");
1718 return cleanup(ohci, CLEANUP_PUT_CARD, -ENODEV);
1719 }
1720
1721 pci_set_master(dev);
1722 pci_write_config_dword(dev, OHCI1394_PCI_HCI_Control, 0);
1723 pci_set_drvdata(dev, ohci);
1724
1725 spin_lock_init(&ohci->lock);
1726
1727 tasklet_init(&ohci->bus_reset_tasklet,
1728 bus_reset_tasklet, (unsigned long)ohci);
1729
1730 if (pci_request_region(dev, 0, ohci_driver_name)) {
1731 fw_error("MMIO resource unavailable\n");
1732 return cleanup(ohci, CLEANUP_DISABLE, -EBUSY);
1733 }
1734
1735 ohci->registers = pci_iomap(dev, 0, OHCI1394_REGISTER_SIZE);
1736 if (ohci->registers == NULL) {
1737 fw_error("Failed to remap registers\n");
1738 return cleanup(ohci, CLEANUP_IOMEM, -ENXIO);
1739 }
1740
1741 if (software_reset(ohci)) {
1742 fw_error("Failed to reset ohci card.\n");
1743 return cleanup(ohci, CLEANUP_REGISTERS, -EBUSY);
1744 }
1745
1746 /* Now enable LPS, which we need in order to start accessing
1747 * most of the registers. In fact, on some cards (ALI M5251),
1748 * accessing registers in the SClk domain without LPS enabled
1749 * will lock up the machine. Wait 50msec to make sure we have
1750 * full link enabled. */
1751 reg_write(ohci, OHCI1394_HCControlSet,
1752 OHCI1394_HCControl_LPS |
1753 OHCI1394_HCControl_postedWriteEnable);
1754 flush_writes(ohci);
1755 msleep(50);
1756
1757 reg_write(ohci, OHCI1394_HCControlClear,
1758 OHCI1394_HCControl_noByteSwapData);
1759
1760 reg_write(ohci, OHCI1394_LinkControlSet,
1761 OHCI1394_LinkControl_rcvSelfID |
1762 OHCI1394_LinkControl_cycleTimerEnable |
1763 OHCI1394_LinkControl_cycleMaster);
1764
1765 ar_context_init(&ohci->ar_request_ctx, ohci,
1766 OHCI1394_AsReqRcvContextControlSet);
1767
1768 ar_context_init(&ohci->ar_response_ctx, ohci,
1769 OHCI1394_AsRspRcvContextControlSet);
1770
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001771 context_init(&ohci->at_request_ctx, ohci, AT_BUFFER_SIZE,
1772 OHCI1394_AsReqTrContextControlSet, handle_at_packet);
Kristian Høgsberged568912006-12-19 19:58:35 -05001773
Kristian Høgsbergf319b6a2007-03-07 12:12:49 -05001774 context_init(&ohci->at_response_ctx, ohci, AT_BUFFER_SIZE,
1775 OHCI1394_AsRspTrContextControlSet, handle_at_packet);
Kristian Høgsberged568912006-12-19 19:58:35 -05001776
1777 reg_write(ohci, OHCI1394_ATRetries,
1778 OHCI1394_MAX_AT_REQ_RETRIES |
1779 (OHCI1394_MAX_AT_RESP_RETRIES << 4) |
1780 (OHCI1394_MAX_PHYS_RESP_RETRIES << 8));
1781
1782 reg_write(ohci, OHCI1394_IsoRecvIntMaskSet, ~0);
1783 ohci->it_context_mask = reg_read(ohci, OHCI1394_IsoRecvIntMaskSet);
1784 reg_write(ohci, OHCI1394_IsoRecvIntMaskClear, ~0);
1785 size = sizeof(struct iso_context) * hweight32(ohci->it_context_mask);
1786 ohci->it_context_list = kzalloc(size, GFP_KERNEL);
1787
1788 reg_write(ohci, OHCI1394_IsoXmitIntMaskSet, ~0);
1789 ohci->ir_context_mask = reg_read(ohci, OHCI1394_IsoXmitIntMaskSet);
1790 reg_write(ohci, OHCI1394_IsoXmitIntMaskClear, ~0);
1791 size = sizeof(struct iso_context) * hweight32(ohci->ir_context_mask);
1792 ohci->ir_context_list = kzalloc(size, GFP_KERNEL);
1793
1794 if (ohci->it_context_list == NULL || ohci->ir_context_list == NULL) {
1795 fw_error("Out of memory for it/ir contexts.\n");
1796 return cleanup(ohci, CLEANUP_REGISTERS, -ENOMEM);
1797 }
1798
1799 /* self-id dma buffer allocation */
1800 ohci->self_id_cpu = dma_alloc_coherent(ohci->card.device,
1801 SELF_ID_BUF_SIZE,
1802 &ohci->self_id_bus,
1803 GFP_KERNEL);
1804 if (ohci->self_id_cpu == NULL) {
1805 fw_error("Out of memory for self ID buffer.\n");
1806 return cleanup(ohci, CLEANUP_REGISTERS, -ENOMEM);
1807 }
1808
1809 reg_write(ohci, OHCI1394_SelfIDBuffer, ohci->self_id_bus);
1810 reg_write(ohci, OHCI1394_PhyUpperBound, 0x00010000);
1811 reg_write(ohci, OHCI1394_IntEventClear, ~0);
1812 reg_write(ohci, OHCI1394_IntMaskClear, ~0);
1813 reg_write(ohci, OHCI1394_IntMaskSet,
1814 OHCI1394_selfIDComplete |
1815 OHCI1394_RQPkt | OHCI1394_RSPkt |
1816 OHCI1394_reqTxComplete | OHCI1394_respTxComplete |
1817 OHCI1394_isochRx | OHCI1394_isochTx |
Kristian Høgsbergd60d7f12007-03-07 12:12:56 -05001818 OHCI1394_masterIntEnable |
1819 OHCI1394_cycle64Seconds);
Kristian Høgsberged568912006-12-19 19:58:35 -05001820
1821 bus_options = reg_read(ohci, OHCI1394_BusOptions);
1822 max_receive = (bus_options >> 12) & 0xf;
1823 link_speed = bus_options & 0x7;
1824 guid = ((u64) reg_read(ohci, OHCI1394_GUIDHi) << 32) |
1825 reg_read(ohci, OHCI1394_GUIDLo);
1826
1827 error_code = fw_card_add(&ohci->card, max_receive, link_speed, guid);
1828 if (error_code < 0)
1829 return cleanup(ohci, CLEANUP_SELF_ID, error_code);
1830
Kristian Høgsberge364cf42007-02-16 17:34:49 -05001831 ohci->version = reg_read(ohci, OHCI1394_Version) & 0x00ff00ff;
Kristian Høgsberg500be722007-02-16 17:34:43 -05001832 fw_notify("Added fw-ohci device %s, OHCI version %x.%x\n",
Kristian Høgsberge364cf42007-02-16 17:34:49 -05001833 dev->dev.bus_id, ohci->version >> 16, ohci->version & 0xff);
Kristian Høgsberged568912006-12-19 19:58:35 -05001834
1835 return 0;
1836}
1837
1838static void pci_remove(struct pci_dev *dev)
1839{
1840 struct fw_ohci *ohci;
1841
1842 ohci = pci_get_drvdata(dev);
Kristian Høgsberge254a4b2007-03-07 12:12:38 -05001843 reg_write(ohci, OHCI1394_IntMaskClear, ~0);
1844 flush_writes(ohci);
Kristian Høgsberged568912006-12-19 19:58:35 -05001845 fw_core_remove_card(&ohci->card);
1846
1847 /* FIXME: Fail all pending packets here, now that the upper
1848 * layers can't queue any more. */
1849
1850 software_reset(ohci);
1851 free_irq(dev->irq, ohci);
1852 cleanup(ohci, CLEANUP_SELF_ID, 0);
1853
1854 fw_notify("Removed fw-ohci device.\n");
1855}
1856
1857static struct pci_device_id pci_table[] = {
1858 { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_FIREWIRE_OHCI, ~0) },
1859 { }
1860};
1861
1862MODULE_DEVICE_TABLE(pci, pci_table);
1863
1864static struct pci_driver fw_ohci_pci_driver = {
1865 .name = ohci_driver_name,
1866 .id_table = pci_table,
1867 .probe = pci_probe,
1868 .remove = pci_remove,
1869};
1870
1871MODULE_AUTHOR("Kristian Hoegsberg <krh@bitplanet.net>");
1872MODULE_DESCRIPTION("Driver for PCI OHCI IEEE1394 controllers");
1873MODULE_LICENSE("GPL");
1874
1875static int __init fw_ohci_init(void)
1876{
1877 return pci_register_driver(&fw_ohci_pci_driver);
1878}
1879
1880static void __exit fw_ohci_cleanup(void)
1881{
1882 pci_unregister_driver(&fw_ohci_pci_driver);
1883}
1884
1885module_init(fw_ohci_init);
1886module_exit(fw_ohci_cleanup);