blob: 7b7ba0e26903d1ffc9570a56738801adac135397 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Jeff Garzikaa7e16d2005-08-29 15:12:56 -04008 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
Linus Torvalds1da177e2005-04-16 15:20:36 -070021 *
Jeff Garzikaf36d7f2005-08-28 20:18:39 -040022 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
Robert Hancockfbbb2622006-10-27 19:08:41 -070032 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 */
38
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/kernel.h>
40#include <linux/module.h>
41#include <linux/pci.h>
42#include <linux/init.h>
43#include <linux/blkdev.h>
44#include <linux/delay.h>
45#include <linux/interrupt.h>
Jeff Garzika9524a72005-10-30 14:39:11 -050046#include <linux/device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#include <scsi/scsi_host.h>
Robert Hancockfbbb2622006-10-27 19:08:41 -070048#include <scsi/scsi_device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049#include <linux/libata.h>
50
51#define DRV_NAME "sata_nv"
Jeff Garzik2a3103c2007-08-31 04:54:06 -040052#define DRV_VERSION "3.5"
Robert Hancockfbbb2622006-10-27 19:08:41 -070053
54#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
Jeff Garzik10ad05d2006-03-22 23:50:50 -050056enum {
Tejun Heo0d5ff562007-02-01 15:06:36 +090057 NV_MMIO_BAR = 5,
58
Jeff Garzik10ad05d2006-03-22 23:50:50 -050059 NV_PORTS = 2,
60 NV_PIO_MASK = 0x1f,
61 NV_MWDMA_MASK = 0x07,
62 NV_UDMA_MASK = 0x7f,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
Tejun Heo27e4b272006-06-17 15:49:55 +090066 /* INT_STATUS/ENABLE */
Jeff Garzik10ad05d2006-03-22 23:50:50 -050067 NV_INT_STATUS = 0x10,
Jeff Garzik10ad05d2006-03-22 23:50:50 -050068 NV_INT_ENABLE = 0x11,
Tejun Heo27e4b272006-06-17 15:49:55 +090069 NV_INT_STATUS_CK804 = 0x440,
Jeff Garzik10ad05d2006-03-22 23:50:50 -050070 NV_INT_ENABLE_CK804 = 0x441,
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Tejun Heo27e4b272006-06-17 15:49:55 +090072 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
Tejun Heo39f87582006-06-17 15:49:56 +090080 NV_INT_ALL = 0x0f,
Tejun Heo5a44eff2006-06-17 15:49:56 +090081 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
Tejun Heo39f87582006-06-17 15:49:56 +090083
Tejun Heo27e4b272006-06-17 15:49:55 +090084 /* INT_CONFIG */
Jeff Garzik10ad05d2006-03-22 23:50:50 -050085 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
Jeff Garzik10ad05d2006-03-22 23:50:50 -050088 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
Robert Hancockfbbb2622006-10-27 19:08:41 -070091 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400166 NV_ADMA_STAT_TIMEOUT,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
Robert Hancock2dec7552006-11-26 14:20:19 -0600170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
Robert Hancockfbbb2622006-10-27 19:08:41 -0700171
Kuan Luof140f0f2007-10-15 15:16:53 -0400172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
Jeff Garzik10ad05d2006-03-22 23:50:50 -0500201};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202
Robert Hancockfbbb2622006-10-27 19:08:41 -0700203/* ADMA Physical Region Descriptor - one SG segment */
204struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210};
211
212enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220};
221
222/* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400231 u8 len; /* 3 */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
239};
240
241
242struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
Robert Hancock8959d302008-02-04 19:39:02 -0600250 u64 adma_dma_mask;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700251 u8 flags;
Robert Hancock5e5c74a2007-02-19 18:42:30 -0600252 int last_issue_ncq;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700253};
254
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600255struct nv_host_priv {
256 unsigned long type;
257};
258
Kuan Luof140f0f2007-10-15 15:16:53 -0400259struct defer_queue {
260 u32 defer_bits;
261 unsigned int head;
262 unsigned int tail;
263 unsigned int tag[ATA_MAX_QUEUE];
264};
265
266enum ncq_saw_flag_list {
267 ncq_saw_d2h = (1U << 0),
268 ncq_saw_dmas = (1U << 1),
269 ncq_saw_sdb = (1U << 2),
270 ncq_saw_backout = (1U << 3),
271};
272
273struct nv_swncq_port_priv {
274 struct ata_prd *prd; /* our SG list */
275 dma_addr_t prd_dma; /* and its DMA mapping */
276 void __iomem *sactive_block;
277 void __iomem *irq_block;
278 void __iomem *tag_block;
279 u32 qc_active;
280
281 unsigned int last_issue_tag;
282
283 /* fifo circular queue to store deferral command */
284 struct defer_queue defer_queue;
285
286 /* for NCQ interrupt analysis */
287 u32 dhfis_bits;
288 u32 dmafis_bits;
289 u32 sdbfis_bits;
290
291 unsigned int ncq_flags;
292};
293
294
Jeff Garzik5796d1c2007-10-26 00:03:37 -0400295#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
Robert Hancockfbbb2622006-10-27 19:08:41 -0700296
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400297static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900298#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600299static int nv_pci_device_resume(struct pci_dev *pdev);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900300#endif
Jeff Garzikcca39742006-08-24 03:19:22 -0400301static void nv_ck804_host_stop(struct ata_host *host);
David Howells7d12e782006-10-05 14:55:46 +0100302static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
303static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
304static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400305static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
306static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
Tejun Heo39f87582006-06-17 15:49:56 +0900308static void nv_nf2_freeze(struct ata_port *ap);
309static void nv_nf2_thaw(struct ata_port *ap);
310static void nv_ck804_freeze(struct ata_port *ap);
311static void nv_ck804_thaw(struct ata_port *ap);
312static void nv_error_handler(struct ata_port *ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700313static int nv_adma_slave_config(struct scsi_device *sdev);
Robert Hancock2dec7552006-11-26 14:20:19 -0600314static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700315static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
316static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
317static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
318static void nv_adma_irq_clear(struct ata_port *ap);
319static int nv_adma_port_start(struct ata_port *ap);
320static void nv_adma_port_stop(struct ata_port *ap);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900321#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600322static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
323static int nv_adma_port_resume(struct ata_port *ap);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900324#endif
Robert Hancock53014e22007-05-05 15:36:36 -0600325static void nv_adma_freeze(struct ata_port *ap);
326static void nv_adma_thaw(struct ata_port *ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700327static void nv_adma_error_handler(struct ata_port *ap);
328static void nv_adma_host_stop(struct ata_host *host);
Robert Hancockf5ecac22007-02-20 21:49:10 -0600329static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
Robert Hancockf2fb3442007-03-26 21:43:36 -0800330static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
Tejun Heo39f87582006-06-17 15:49:56 +0900331
Kuan Luof140f0f2007-10-15 15:16:53 -0400332static void nv_mcp55_thaw(struct ata_port *ap);
333static void nv_mcp55_freeze(struct ata_port *ap);
334static void nv_swncq_error_handler(struct ata_port *ap);
335static int nv_swncq_slave_config(struct scsi_device *sdev);
336static int nv_swncq_port_start(struct ata_port *ap);
337static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
338static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
339static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
340static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
341static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
342#ifdef CONFIG_PM
343static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
344static int nv_swncq_port_resume(struct ata_port *ap);
345#endif
346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347enum nv_host_type
348{
349 GENERIC,
350 NFORCE2,
Tejun Heo27e4b272006-06-17 15:49:55 +0900351 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700352 CK804,
Kuan Luof140f0f2007-10-15 15:16:53 -0400353 ADMA,
354 SWNCQ,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355};
356
Jeff Garzik3b7d6972005-11-10 11:04:11 -0500357static const struct pci_device_id nv_pci_tbl[] = {
Jeff Garzik54bb3a92006-09-27 22:20:11 -0400358 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
359 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
Kuan Luof140f0f2007-10-15 15:16:53 -0400365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), SWNCQ },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), SWNCQ },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), SWNCQ },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), SWNCQ },
Kuan Luoe2e031e2007-10-25 02:14:17 -0400369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
Jeff Garzik2d2744f2006-09-28 20:21:59 -0400372
373 { } /* terminate list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374};
375
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376static struct pci_driver nv_pci_driver = {
377 .name = DRV_NAME,
378 .id_table = nv_pci_tbl,
379 .probe = nv_init_one,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900380#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600381 .suspend = ata_pci_device_suspend,
382 .resume = nv_pci_device_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900383#endif
Tejun Heo1daf9ce2007-05-17 13:13:57 +0200384 .remove = ata_pci_remove_one,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385};
386
Jeff Garzik193515d2005-11-07 00:59:37 -0500387static struct scsi_host_template nv_sht = {
Tejun Heo68d1d072008-03-25 12:22:49 +0900388 ATA_BMDMA_SHT(DRV_NAME),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389};
390
Robert Hancockfbbb2622006-10-27 19:08:41 -0700391static struct scsi_host_template nv_adma_sht = {
Tejun Heo68d1d072008-03-25 12:22:49 +0900392 ATA_NCQ_SHT(DRV_NAME),
Robert Hancockfbbb2622006-10-27 19:08:41 -0700393 .can_queue = NV_ADMA_MAX_CPBS,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700394 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700395 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
396 .slave_configure = nv_adma_slave_config,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700397};
398
Kuan Luof140f0f2007-10-15 15:16:53 -0400399static struct scsi_host_template nv_swncq_sht = {
Tejun Heo68d1d072008-03-25 12:22:49 +0900400 ATA_NCQ_SHT(DRV_NAME),
Kuan Luof140f0f2007-10-15 15:16:53 -0400401 .can_queue = ATA_MAX_QUEUE,
Kuan Luof140f0f2007-10-15 15:16:53 -0400402 .sg_tablesize = LIBATA_MAX_PRD,
Kuan Luof140f0f2007-10-15 15:16:53 -0400403 .dma_boundary = ATA_DMA_BOUNDARY,
404 .slave_configure = nv_swncq_slave_config,
Kuan Luof140f0f2007-10-15 15:16:53 -0400405};
406
Tejun Heo029cfd62008-03-25 12:22:49 +0900407static struct ata_port_operations nv_generic_ops = {
408 .inherits = &ata_bmdma_port_ops,
Tejun Heo39f87582006-06-17 15:49:56 +0900409 .error_handler = nv_error_handler,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 .scr_read = nv_scr_read,
411 .scr_write = nv_scr_write,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412};
413
Tejun Heo029cfd62008-03-25 12:22:49 +0900414static struct ata_port_operations nv_nf2_ops = {
415 .inherits = &nv_generic_ops,
Tejun Heo39f87582006-06-17 15:49:56 +0900416 .freeze = nv_nf2_freeze,
417 .thaw = nv_nf2_thaw,
Tejun Heoada364e2006-06-17 15:49:56 +0900418};
419
Tejun Heo029cfd62008-03-25 12:22:49 +0900420static struct ata_port_operations nv_ck804_ops = {
421 .inherits = &nv_generic_ops,
Tejun Heo39f87582006-06-17 15:49:56 +0900422 .freeze = nv_ck804_freeze,
423 .thaw = nv_ck804_thaw,
Tejun Heoada364e2006-06-17 15:49:56 +0900424 .host_stop = nv_ck804_host_stop,
425};
426
Tejun Heo029cfd62008-03-25 12:22:49 +0900427static struct ata_port_operations nv_adma_ops = {
428 .inherits = &nv_generic_ops,
429
Robert Hancock2dec7552006-11-26 14:20:19 -0600430 .check_atapi_dma = nv_adma_check_atapi_dma,
Tejun Heo029cfd62008-03-25 12:22:49 +0900431 .tf_read = nv_adma_tf_read,
Tejun Heo31cc23b2007-09-23 13:14:12 +0900432 .qc_defer = ata_std_qc_defer,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700433 .qc_prep = nv_adma_qc_prep,
434 .qc_issue = nv_adma_qc_issue,
Tejun Heo029cfd62008-03-25 12:22:49 +0900435 .irq_clear = nv_adma_irq_clear,
436
Robert Hancock53014e22007-05-05 15:36:36 -0600437 .freeze = nv_adma_freeze,
438 .thaw = nv_adma_thaw,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700439 .error_handler = nv_adma_error_handler,
Robert Hancockf5ecac22007-02-20 21:49:10 -0600440 .post_internal_cmd = nv_adma_post_internal_cmd,
Tejun Heo029cfd62008-03-25 12:22:49 +0900441
Robert Hancockfbbb2622006-10-27 19:08:41 -0700442 .port_start = nv_adma_port_start,
443 .port_stop = nv_adma_port_stop,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900444#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600445 .port_suspend = nv_adma_port_suspend,
446 .port_resume = nv_adma_port_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900447#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -0700448 .host_stop = nv_adma_host_stop,
449};
450
Tejun Heo029cfd62008-03-25 12:22:49 +0900451static struct ata_port_operations nv_swncq_ops = {
452 .inherits = &nv_generic_ops,
453
Kuan Luof140f0f2007-10-15 15:16:53 -0400454 .qc_defer = ata_std_qc_defer,
455 .qc_prep = nv_swncq_qc_prep,
456 .qc_issue = nv_swncq_qc_issue,
Tejun Heo029cfd62008-03-25 12:22:49 +0900457
Kuan Luof140f0f2007-10-15 15:16:53 -0400458 .freeze = nv_mcp55_freeze,
459 .thaw = nv_mcp55_thaw,
460 .error_handler = nv_swncq_error_handler,
Tejun Heo029cfd62008-03-25 12:22:49 +0900461
Kuan Luof140f0f2007-10-15 15:16:53 -0400462#ifdef CONFIG_PM
463 .port_suspend = nv_swncq_port_suspend,
464 .port_resume = nv_swncq_port_resume,
465#endif
466 .port_start = nv_swncq_port_start,
467};
468
Tejun Heo1626aeb2007-05-04 12:43:58 +0200469static const struct ata_port_info nv_port_info[] = {
Tejun Heoada364e2006-06-17 15:49:56 +0900470 /* generic */
471 {
472 .sht = &nv_sht,
Tejun Heo0c887582007-08-06 18:36:23 +0900473 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
Tejun Heoada364e2006-06-17 15:49:56 +0900474 .pio_mask = NV_PIO_MASK,
475 .mwdma_mask = NV_MWDMA_MASK,
476 .udma_mask = NV_UDMA_MASK,
477 .port_ops = &nv_generic_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900478 .irq_handler = nv_generic_interrupt,
Tejun Heoada364e2006-06-17 15:49:56 +0900479 },
480 /* nforce2/3 */
481 {
482 .sht = &nv_sht,
Tejun Heo0c887582007-08-06 18:36:23 +0900483 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
Tejun Heoada364e2006-06-17 15:49:56 +0900484 .pio_mask = NV_PIO_MASK,
485 .mwdma_mask = NV_MWDMA_MASK,
486 .udma_mask = NV_UDMA_MASK,
487 .port_ops = &nv_nf2_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900488 .irq_handler = nv_nf2_interrupt,
Tejun Heoada364e2006-06-17 15:49:56 +0900489 },
490 /* ck804 */
491 {
492 .sht = &nv_sht,
Tejun Heo0c887582007-08-06 18:36:23 +0900493 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
Tejun Heoada364e2006-06-17 15:49:56 +0900494 .pio_mask = NV_PIO_MASK,
495 .mwdma_mask = NV_MWDMA_MASK,
496 .udma_mask = NV_UDMA_MASK,
497 .port_ops = &nv_ck804_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900498 .irq_handler = nv_ck804_interrupt,
Tejun Heoada364e2006-06-17 15:49:56 +0900499 },
Robert Hancockfbbb2622006-10-27 19:08:41 -0700500 /* ADMA */
501 {
502 .sht = &nv_adma_sht,
503 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
504 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
505 .pio_mask = NV_PIO_MASK,
506 .mwdma_mask = NV_MWDMA_MASK,
507 .udma_mask = NV_UDMA_MASK,
508 .port_ops = &nv_adma_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900509 .irq_handler = nv_adma_interrupt,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700510 },
Kuan Luof140f0f2007-10-15 15:16:53 -0400511 /* SWNCQ */
512 {
513 .sht = &nv_swncq_sht,
514 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
515 ATA_FLAG_NCQ,
Kuan Luof140f0f2007-10-15 15:16:53 -0400516 .pio_mask = NV_PIO_MASK,
517 .mwdma_mask = NV_MWDMA_MASK,
518 .udma_mask = NV_UDMA_MASK,
519 .port_ops = &nv_swncq_ops,
520 .irq_handler = nv_swncq_interrupt,
521 },
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522};
523
524MODULE_AUTHOR("NVIDIA");
525MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
526MODULE_LICENSE("GPL");
527MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
528MODULE_VERSION(DRV_VERSION);
529
Robert Hancockfbbb2622006-10-27 19:08:41 -0700530static int adma_enabled = 1;
Kuan Luof140f0f2007-10-15 15:16:53 -0400531static int swncq_enabled;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700532
Robert Hancock2dec7552006-11-26 14:20:19 -0600533static void nv_adma_register_mode(struct ata_port *ap)
534{
Robert Hancock2dec7552006-11-26 14:20:19 -0600535 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600536 void __iomem *mmio = pp->ctl_block;
Robert Hancocka2cfe812007-02-05 16:26:03 -0800537 u16 tmp, status;
538 int count = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600539
540 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
541 return;
542
Robert Hancocka2cfe812007-02-05 16:26:03 -0800543 status = readw(mmio + NV_ADMA_STAT);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400544 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
Robert Hancocka2cfe812007-02-05 16:26:03 -0800545 ndelay(50);
546 status = readw(mmio + NV_ADMA_STAT);
547 count++;
548 }
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400549 if (count == 20)
Robert Hancocka2cfe812007-02-05 16:26:03 -0800550 ata_port_printk(ap, KERN_WARNING,
551 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
552 status);
553
Robert Hancock2dec7552006-11-26 14:20:19 -0600554 tmp = readw(mmio + NV_ADMA_CTL);
555 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
556
Robert Hancocka2cfe812007-02-05 16:26:03 -0800557 count = 0;
558 status = readw(mmio + NV_ADMA_STAT);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400559 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
Robert Hancocka2cfe812007-02-05 16:26:03 -0800560 ndelay(50);
561 status = readw(mmio + NV_ADMA_STAT);
562 count++;
563 }
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400564 if (count == 20)
Robert Hancocka2cfe812007-02-05 16:26:03 -0800565 ata_port_printk(ap, KERN_WARNING,
566 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
567 status);
568
Robert Hancock2dec7552006-11-26 14:20:19 -0600569 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
570}
571
572static void nv_adma_mode(struct ata_port *ap)
573{
Robert Hancock2dec7552006-11-26 14:20:19 -0600574 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600575 void __iomem *mmio = pp->ctl_block;
Robert Hancocka2cfe812007-02-05 16:26:03 -0800576 u16 tmp, status;
577 int count = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600578
579 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
580 return;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500581
Robert Hancock2dec7552006-11-26 14:20:19 -0600582 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
583
584 tmp = readw(mmio + NV_ADMA_CTL);
585 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
586
Robert Hancocka2cfe812007-02-05 16:26:03 -0800587 status = readw(mmio + NV_ADMA_STAT);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400588 while (((status & NV_ADMA_STAT_LEGACY) ||
Robert Hancocka2cfe812007-02-05 16:26:03 -0800589 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
590 ndelay(50);
591 status = readw(mmio + NV_ADMA_STAT);
592 count++;
593 }
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400594 if (count == 20)
Robert Hancocka2cfe812007-02-05 16:26:03 -0800595 ata_port_printk(ap, KERN_WARNING,
596 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
597 status);
598
Robert Hancock2dec7552006-11-26 14:20:19 -0600599 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
600}
601
Robert Hancockfbbb2622006-10-27 19:08:41 -0700602static int nv_adma_slave_config(struct scsi_device *sdev)
603{
604 struct ata_port *ap = ata_shost_to_port(sdev->host);
Robert Hancock2dec7552006-11-26 14:20:19 -0600605 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancock8959d302008-02-04 19:39:02 -0600606 struct nv_adma_port_priv *port0, *port1;
607 struct scsi_device *sdev0, *sdev1;
Robert Hancock2dec7552006-11-26 14:20:19 -0600608 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
Robert Hancock8959d302008-02-04 19:39:02 -0600609 unsigned long segment_boundary, flags;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700610 unsigned short sg_tablesize;
611 int rc;
Robert Hancock2dec7552006-11-26 14:20:19 -0600612 int adma_enable;
613 u32 current_reg, new_reg, config_mask;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700614
615 rc = ata_scsi_slave_config(sdev);
616
617 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
618 /* Not a proper libata device, ignore */
619 return rc;
620
Robert Hancock8959d302008-02-04 19:39:02 -0600621 spin_lock_irqsave(ap->lock, flags);
622
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900623 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
Robert Hancockfbbb2622006-10-27 19:08:41 -0700624 /*
625 * NVIDIA reports that ADMA mode does not support ATAPI commands.
626 * Therefore ATAPI commands are sent through the legacy interface.
627 * However, the legacy interface only supports 32-bit DMA.
628 * Restrict DMA parameters as required by the legacy interface
629 * when an ATAPI device is connected.
630 */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700631 segment_boundary = ATA_DMA_BOUNDARY;
632 /* Subtract 1 since an extra entry may be needed for padding, see
633 libata-scsi.c */
634 sg_tablesize = LIBATA_MAX_PRD - 1;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500635
Robert Hancock2dec7552006-11-26 14:20:19 -0600636 /* Since the legacy DMA engine is in use, we need to disable ADMA
637 on the port. */
638 adma_enable = 0;
639 nv_adma_register_mode(ap);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400640 } else {
Robert Hancockfbbb2622006-10-27 19:08:41 -0700641 segment_boundary = NV_ADMA_DMA_BOUNDARY;
642 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
Robert Hancock2dec7552006-11-26 14:20:19 -0600643 adma_enable = 1;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700644 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500645
Robert Hancock2dec7552006-11-26 14:20:19 -0600646 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700647
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400648 if (ap->port_no == 1)
Robert Hancock2dec7552006-11-26 14:20:19 -0600649 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
650 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
651 else
652 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
653 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500654
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400655 if (adma_enable) {
Robert Hancock2dec7552006-11-26 14:20:19 -0600656 new_reg = current_reg | config_mask;
657 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400658 } else {
Robert Hancock2dec7552006-11-26 14:20:19 -0600659 new_reg = current_reg & ~config_mask;
660 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
661 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500662
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400663 if (current_reg != new_reg)
Robert Hancock2dec7552006-11-26 14:20:19 -0600664 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500665
Robert Hancock8959d302008-02-04 19:39:02 -0600666 port0 = ap->host->ports[0]->private_data;
667 port1 = ap->host->ports[1]->private_data;
668 sdev0 = ap->host->ports[0]->link.device[0].sdev;
669 sdev1 = ap->host->ports[1]->link.device[0].sdev;
670 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
671 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
672 /** We have to set the DMA mask to 32-bit if either port is in
673 ATAPI mode, since they are on the same PCI device which is
674 used for DMA mapping. If we set the mask we also need to set
675 the bounce limit on both ports to ensure that the block
676 layer doesn't feed addresses that cause DMA mapping to
677 choke. If either SCSI device is not allocated yet, it's OK
678 since that port will discover its correct setting when it
679 does get allocated.
680 Note: Setting 32-bit mask should not fail. */
681 if (sdev0)
682 blk_queue_bounce_limit(sdev0->request_queue,
683 ATA_DMA_MASK);
684 if (sdev1)
685 blk_queue_bounce_limit(sdev1->request_queue,
686 ATA_DMA_MASK);
687
688 pci_set_dma_mask(pdev, ATA_DMA_MASK);
689 } else {
690 /** This shouldn't fail as it was set to this value before */
691 pci_set_dma_mask(pdev, pp->adma_dma_mask);
692 if (sdev0)
693 blk_queue_bounce_limit(sdev0->request_queue,
694 pp->adma_dma_mask);
695 if (sdev1)
696 blk_queue_bounce_limit(sdev1->request_queue,
697 pp->adma_dma_mask);
698 }
699
Robert Hancockfbbb2622006-10-27 19:08:41 -0700700 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
701 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
702 ata_port_printk(ap, KERN_INFO,
Robert Hancock8959d302008-02-04 19:39:02 -0600703 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
704 (unsigned long long)*ap->host->dev->dma_mask,
705 segment_boundary, sg_tablesize);
706
707 spin_unlock_irqrestore(ap->lock, flags);
708
Robert Hancockfbbb2622006-10-27 19:08:41 -0700709 return rc;
710}
711
Robert Hancock2dec7552006-11-26 14:20:19 -0600712static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
713{
714 struct nv_adma_port_priv *pp = qc->ap->private_data;
715 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
716}
717
Robert Hancockf2fb3442007-03-26 21:43:36 -0800718static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
719{
Robert Hancock3f3debd2007-11-25 16:59:36 -0600720 /* Other than when internal or pass-through commands are executed,
721 the only time this function will be called in ADMA mode will be
722 if a command fails. In the failure case we don't care about going
723 into register mode with ADMA commands pending, as the commands will
724 all shortly be aborted anyway. We assume that NCQ commands are not
725 issued via passthrough, which is the only way that switching into
726 ADMA mode could abort outstanding commands. */
Robert Hancockf2fb3442007-03-26 21:43:36 -0800727 nv_adma_register_mode(ap);
728
729 ata_tf_read(ap, tf);
730}
731
Robert Hancock2dec7552006-11-26 14:20:19 -0600732static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
Robert Hancockfbbb2622006-10-27 19:08:41 -0700733{
734 unsigned int idx = 0;
735
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400736 if (tf->flags & ATA_TFLAG_ISADDR) {
Robert Hancockac3d6b82007-02-19 19:02:46 -0600737 if (tf->flags & ATA_TFLAG_LBA48) {
738 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
739 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
740 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
741 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
742 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
743 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
744 } else
745 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
Jeff Garzika84471f2007-02-26 05:51:33 -0500746
Robert Hancockac3d6b82007-02-19 19:02:46 -0600747 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
748 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
749 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
750 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700751 }
Jeff Garzika84471f2007-02-26 05:51:33 -0500752
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400753 if (tf->flags & ATA_TFLAG_DEVICE)
Robert Hancockac3d6b82007-02-19 19:02:46 -0600754 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700755
756 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
Jeff Garzika84471f2007-02-26 05:51:33 -0500757
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400758 while (idx < 12)
Robert Hancockac3d6b82007-02-19 19:02:46 -0600759 cpb[idx++] = cpu_to_le16(IGN);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700760
761 return idx;
762}
763
Robert Hancock5bd28a42007-02-05 16:26:01 -0800764static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
Robert Hancockfbbb2622006-10-27 19:08:41 -0700765{
766 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancock2dec7552006-11-26 14:20:19 -0600767 u8 flags = pp->cpb[cpb_num].resp_flags;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700768
769 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
770
Robert Hancock5bd28a42007-02-05 16:26:01 -0800771 if (unlikely((force_err ||
772 flags & (NV_CPB_RESP_ATA_ERR |
773 NV_CPB_RESP_CMD_ERR |
774 NV_CPB_RESP_CPB_ERR)))) {
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900775 struct ata_eh_info *ehi = &ap->link.eh_info;
Robert Hancock5bd28a42007-02-05 16:26:01 -0800776 int freeze = 0;
777
778 ata_ehi_clear_desc(ehi);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400779 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
Robert Hancock5bd28a42007-02-05 16:26:01 -0800780 if (flags & NV_CPB_RESP_ATA_ERR) {
Tejun Heob64bbc32007-07-16 14:29:39 +0900781 ata_ehi_push_desc(ehi, "ATA error");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800782 ehi->err_mask |= AC_ERR_DEV;
783 } else if (flags & NV_CPB_RESP_CMD_ERR) {
Tejun Heob64bbc32007-07-16 14:29:39 +0900784 ata_ehi_push_desc(ehi, "CMD error");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800785 ehi->err_mask |= AC_ERR_DEV;
786 } else if (flags & NV_CPB_RESP_CPB_ERR) {
Tejun Heob64bbc32007-07-16 14:29:39 +0900787 ata_ehi_push_desc(ehi, "CPB error");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800788 ehi->err_mask |= AC_ERR_SYSTEM;
789 freeze = 1;
790 } else {
791 /* notifier error, but no error in CPB flags? */
Tejun Heob64bbc32007-07-16 14:29:39 +0900792 ata_ehi_push_desc(ehi, "unknown");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800793 ehi->err_mask |= AC_ERR_OTHER;
794 freeze = 1;
795 }
796 /* Kill all commands. EH will determine what actually failed. */
797 if (freeze)
798 ata_port_freeze(ap);
799 else
800 ata_port_abort(ap);
801 return 1;
802 }
803
Robert Hancockf2fb3442007-03-26 21:43:36 -0800804 if (likely(flags & NV_CPB_RESP_DONE)) {
Robert Hancockfbbb2622006-10-27 19:08:41 -0700805 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
Robert Hancock5bd28a42007-02-05 16:26:01 -0800806 VPRINTK("CPB flags done, flags=0x%x\n", flags);
807 if (likely(qc)) {
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400808 DPRINTK("Completing qc from tag %d\n", cpb_num);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700809 ata_qc_complete(qc);
Robert Hancock2a54cf72007-02-21 23:53:03 -0600810 } else {
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900811 struct ata_eh_info *ehi = &ap->link.eh_info;
Robert Hancock2a54cf72007-02-21 23:53:03 -0600812 /* Notifier bits set without a command may indicate the drive
813 is misbehaving. Raise host state machine violation on this
814 condition. */
Jeff Garzik5796d1c2007-10-26 00:03:37 -0400815 ata_port_printk(ap, KERN_ERR,
816 "notifier for tag %d with no cmd?\n",
817 cpb_num);
Robert Hancock2a54cf72007-02-21 23:53:03 -0600818 ehi->err_mask |= AC_ERR_HSM;
Tejun Heocf480622008-01-24 00:05:14 +0900819 ehi->action |= ATA_EH_RESET;
Robert Hancock2a54cf72007-02-21 23:53:03 -0600820 ata_port_freeze(ap);
821 return 1;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700822 }
823 }
Robert Hancock5bd28a42007-02-05 16:26:01 -0800824 return 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700825}
826
Robert Hancock2dec7552006-11-26 14:20:19 -0600827static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
828{
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900829 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
Robert Hancock2dec7552006-11-26 14:20:19 -0600830
831 /* freeze if hotplugged */
832 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
833 ata_port_freeze(ap);
834 return 1;
835 }
836
837 /* bail out if not our interrupt */
838 if (!(irq_stat & NV_INT_DEV))
839 return 0;
840
841 /* DEV interrupt w/ no active qc? */
842 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
843 ata_check_status(ap);
844 return 1;
845 }
846
847 /* handle interrupt */
Robert Hancockf740d162007-01-23 20:09:02 -0600848 return ata_host_intr(ap, qc);
Robert Hancock2dec7552006-11-26 14:20:19 -0600849}
850
Robert Hancockfbbb2622006-10-27 19:08:41 -0700851static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
852{
853 struct ata_host *host = dev_instance;
854 int i, handled = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600855 u32 notifier_clears[2];
Robert Hancockfbbb2622006-10-27 19:08:41 -0700856
857 spin_lock(&host->lock);
858
859 for (i = 0; i < host->n_ports; i++) {
860 struct ata_port *ap = host->ports[i];
Robert Hancock2dec7552006-11-26 14:20:19 -0600861 notifier_clears[i] = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700862
863 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
864 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600865 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700866 u16 status;
867 u32 gen_ctl;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700868 u32 notifier, notifier_error;
Jeff Garzika617c092007-05-21 20:14:23 -0400869
Robert Hancock53014e22007-05-05 15:36:36 -0600870 /* if ADMA is disabled, use standard ata interrupt handler */
871 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
872 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
873 >> (NV_INT_PORT_SHIFT * i);
874 handled += nv_host_intr(ap, irq_stat);
875 continue;
876 }
Robert Hancockfbbb2622006-10-27 19:08:41 -0700877
Robert Hancock53014e22007-05-05 15:36:36 -0600878 /* if in ATA register mode, check for standard interrupts */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700879 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
Tejun Heo0d5ff562007-02-01 15:06:36 +0900880 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
Robert Hancock2dec7552006-11-26 14:20:19 -0600881 >> (NV_INT_PORT_SHIFT * i);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400882 if (ata_tag_valid(ap->link.active_tag))
Robert Hancockf740d162007-01-23 20:09:02 -0600883 /** NV_INT_DEV indication seems unreliable at times
884 at least in ADMA mode. Force it on always when a
885 command is active, to prevent losing interrupts. */
886 irq_stat |= NV_INT_DEV;
Robert Hancock2dec7552006-11-26 14:20:19 -0600887 handled += nv_host_intr(ap, irq_stat);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700888 }
889
890 notifier = readl(mmio + NV_ADMA_NOTIFIER);
891 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
Robert Hancock2dec7552006-11-26 14:20:19 -0600892 notifier_clears[i] = notifier | notifier_error;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700893
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600894 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700895
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400896 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
Robert Hancockfbbb2622006-10-27 19:08:41 -0700897 !notifier_error)
898 /* Nothing to do */
899 continue;
900
901 status = readw(mmio + NV_ADMA_STAT);
902
903 /* Clear status. Ensure the controller sees the clearing before we start
904 looking at any of the CPB statuses, so that any CPB completions after
905 this point in the handler will raise another interrupt. */
906 writew(status, mmio + NV_ADMA_STAT);
907 readw(mmio + NV_ADMA_STAT); /* flush posted write */
908 rmb();
909
Robert Hancock5bd28a42007-02-05 16:26:01 -0800910 handled++; /* irq handled if we got here */
911
912 /* freeze if hotplugged or controller error */
913 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
914 NV_ADMA_STAT_HOTUNPLUG |
Robert Hancock5278b502007-02-11 18:36:56 -0600915 NV_ADMA_STAT_TIMEOUT |
916 NV_ADMA_STAT_SERROR))) {
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900917 struct ata_eh_info *ehi = &ap->link.eh_info;
Robert Hancock5bd28a42007-02-05 16:26:01 -0800918
919 ata_ehi_clear_desc(ehi);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400920 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
Robert Hancock5bd28a42007-02-05 16:26:01 -0800921 if (status & NV_ADMA_STAT_TIMEOUT) {
922 ehi->err_mask |= AC_ERR_SYSTEM;
Tejun Heob64bbc32007-07-16 14:29:39 +0900923 ata_ehi_push_desc(ehi, "timeout");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800924 } else if (status & NV_ADMA_STAT_HOTPLUG) {
925 ata_ehi_hotplugged(ehi);
Tejun Heob64bbc32007-07-16 14:29:39 +0900926 ata_ehi_push_desc(ehi, "hotplug");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800927 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
928 ata_ehi_hotplugged(ehi);
Tejun Heob64bbc32007-07-16 14:29:39 +0900929 ata_ehi_push_desc(ehi, "hot unplug");
Robert Hancock5278b502007-02-11 18:36:56 -0600930 } else if (status & NV_ADMA_STAT_SERROR) {
931 /* let libata analyze SError and figure out the cause */
Tejun Heob64bbc32007-07-16 14:29:39 +0900932 ata_ehi_push_desc(ehi, "SError");
933 } else
934 ata_ehi_push_desc(ehi, "unknown");
Robert Hancockfbbb2622006-10-27 19:08:41 -0700935 ata_port_freeze(ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700936 continue;
937 }
938
Robert Hancock5bd28a42007-02-05 16:26:01 -0800939 if (status & (NV_ADMA_STAT_DONE |
Robert Hancocka1fe7822008-01-29 19:53:19 -0600940 NV_ADMA_STAT_CPBERR |
941 NV_ADMA_STAT_CMD_COMPLETE)) {
942 u32 check_commands = notifier_clears[i];
Robert Hancock721449b2007-02-19 19:03:08 -0600943 int pos, error = 0;
Robert Hancock8ba5e4c2007-03-08 18:02:18 -0600944
Robert Hancocka1fe7822008-01-29 19:53:19 -0600945 if (status & NV_ADMA_STAT_CPBERR) {
946 /* Check all active commands */
947 if (ata_tag_valid(ap->link.active_tag))
948 check_commands = 1 <<
949 ap->link.active_tag;
950 else
951 check_commands = ap->
952 link.sactive;
953 }
Robert Hancock8ba5e4c2007-03-08 18:02:18 -0600954
Robert Hancockfbbb2622006-10-27 19:08:41 -0700955 /** Check CPBs for completed commands */
Robert Hancock721449b2007-02-19 19:03:08 -0600956 while ((pos = ffs(check_commands)) && !error) {
957 pos--;
958 error = nv_adma_check_cpb(ap, pos,
Jeff Garzik5796d1c2007-10-26 00:03:37 -0400959 notifier_error & (1 << pos));
960 check_commands &= ~(1 << pos);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700961 }
962 }
Robert Hancockfbbb2622006-10-27 19:08:41 -0700963 }
964 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500965
Jeff Garzikb4479162007-10-25 20:47:30 -0400966 if (notifier_clears[0] || notifier_clears[1]) {
Robert Hancock2dec7552006-11-26 14:20:19 -0600967 /* Note: Both notifier clear registers must be written
968 if either is set, even if one is zero, according to NVIDIA. */
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600969 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
970 writel(notifier_clears[0], pp->notifier_clear_block);
971 pp = host->ports[1]->private_data;
972 writel(notifier_clears[1], pp->notifier_clear_block);
Robert Hancock2dec7552006-11-26 14:20:19 -0600973 }
Robert Hancockfbbb2622006-10-27 19:08:41 -0700974
975 spin_unlock(&host->lock);
976
977 return IRQ_RETVAL(handled);
978}
979
Robert Hancock53014e22007-05-05 15:36:36 -0600980static void nv_adma_freeze(struct ata_port *ap)
981{
982 struct nv_adma_port_priv *pp = ap->private_data;
983 void __iomem *mmio = pp->ctl_block;
984 u16 tmp;
985
986 nv_ck804_freeze(ap);
987
988 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
989 return;
990
991 /* clear any outstanding CK804 notifications */
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400992 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
Robert Hancock53014e22007-05-05 15:36:36 -0600993 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
994
995 /* Disable interrupt */
996 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400997 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
Robert Hancock53014e22007-05-05 15:36:36 -0600998 mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -0400999 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancock53014e22007-05-05 15:36:36 -06001000}
1001
1002static void nv_adma_thaw(struct ata_port *ap)
1003{
1004 struct nv_adma_port_priv *pp = ap->private_data;
1005 void __iomem *mmio = pp->ctl_block;
1006 u16 tmp;
1007
1008 nv_ck804_thaw(ap);
1009
1010 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1011 return;
1012
1013 /* Enable interrupt */
1014 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik2dcb4072007-10-19 06:42:56 -04001015 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
Robert Hancock53014e22007-05-05 15:36:36 -06001016 mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001017 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancock53014e22007-05-05 15:36:36 -06001018}
1019
Robert Hancockfbbb2622006-10-27 19:08:41 -07001020static void nv_adma_irq_clear(struct ata_port *ap)
1021{
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001022 struct nv_adma_port_priv *pp = ap->private_data;
1023 void __iomem *mmio = pp->ctl_block;
Robert Hancock53014e22007-05-05 15:36:36 -06001024 u32 notifier_clears[2];
1025
1026 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1027 ata_bmdma_irq_clear(ap);
1028 return;
1029 }
1030
1031 /* clear any outstanding CK804 notifications */
Jeff Garzik2dcb4072007-10-19 06:42:56 -04001032 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
Robert Hancock53014e22007-05-05 15:36:36 -06001033 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001034
1035 /* clear ADMA status */
Robert Hancock53014e22007-05-05 15:36:36 -06001036 writew(0xffff, mmio + NV_ADMA_STAT);
Jeff Garzika617c092007-05-21 20:14:23 -04001037
Robert Hancock53014e22007-05-05 15:36:36 -06001038 /* clear notifiers - note both ports need to be written with
1039 something even though we are only clearing on one */
1040 if (ap->port_no == 0) {
1041 notifier_clears[0] = 0xFFFFFFFF;
1042 notifier_clears[1] = 0;
1043 } else {
1044 notifier_clears[0] = 0;
1045 notifier_clears[1] = 0xFFFFFFFF;
1046 }
1047 pp = ap->host->ports[0]->private_data;
1048 writel(notifier_clears[0], pp->notifier_clear_block);
1049 pp = ap->host->ports[1]->private_data;
1050 writel(notifier_clears[1], pp->notifier_clear_block);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001051}
1052
Robert Hancockf5ecac22007-02-20 21:49:10 -06001053static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001054{
Robert Hancockf5ecac22007-02-20 21:49:10 -06001055 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001056
Jeff Garzikb4479162007-10-25 20:47:30 -04001057 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
Robert Hancockf5ecac22007-02-20 21:49:10 -06001058 ata_bmdma_post_internal_cmd(qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001059}
1060
1061static int nv_adma_port_start(struct ata_port *ap)
1062{
1063 struct device *dev = ap->host->dev;
1064 struct nv_adma_port_priv *pp;
1065 int rc;
1066 void *mem;
1067 dma_addr_t mem_dma;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001068 void __iomem *mmio;
Robert Hancock8959d302008-02-04 19:39:02 -06001069 struct pci_dev *pdev = to_pci_dev(dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001070 u16 tmp;
1071
1072 VPRINTK("ENTER\n");
1073
Robert Hancock8959d302008-02-04 19:39:02 -06001074 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1075 pad buffers */
1076 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1077 if (rc)
1078 return rc;
1079 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1080 if (rc)
1081 return rc;
1082
Robert Hancockfbbb2622006-10-27 19:08:41 -07001083 rc = ata_port_start(ap);
1084 if (rc)
1085 return rc;
1086
Tejun Heo24dc5f32007-01-20 16:00:28 +09001087 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1088 if (!pp)
1089 return -ENOMEM;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001090
Tejun Heo0d5ff562007-02-01 15:06:36 +09001091 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001092 ap->port_no * NV_ADMA_PORT_SIZE;
1093 pp->ctl_block = mmio;
Tejun Heo0d5ff562007-02-01 15:06:36 +09001094 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001095 pp->notifier_clear_block = pp->gen_block +
1096 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1097
Robert Hancock8959d302008-02-04 19:39:02 -06001098 /* Now that the legacy PRD and padding buffer are allocated we can
1099 safely raise the DMA mask to allocate the CPB/APRD table.
1100 These are allowed to fail since we store the value that ends up
1101 being used to set as the bounce limit in slave_config later if
1102 needed. */
1103 pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1104 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1105 pp->adma_dma_mask = *dev->dma_mask;
1106
Tejun Heo24dc5f32007-01-20 16:00:28 +09001107 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1108 &mem_dma, GFP_KERNEL);
1109 if (!mem)
1110 return -ENOMEM;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001111 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1112
1113 /*
1114 * First item in chunk of DMA memory:
1115 * 128-byte command parameter block (CPB)
1116 * one for each command tag
1117 */
1118 pp->cpb = mem;
1119 pp->cpb_dma = mem_dma;
1120
1121 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001122 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001123
1124 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1125 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1126
1127 /*
1128 * Second item: block of ADMA_SGTBL_LEN s/g entries
1129 */
1130 pp->aprd = mem;
1131 pp->aprd_dma = mem_dma;
1132
1133 ap->private_data = pp;
1134
1135 /* clear any outstanding interrupt conditions */
1136 writew(0xffff, mmio + NV_ADMA_STAT);
1137
1138 /* initialize port variables */
1139 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1140
1141 /* clear CPB fetch count */
1142 writew(0, mmio + NV_ADMA_CPB_COUNT);
1143
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001144 /* clear GO for register mode, enable interrupt */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001145 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001146 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1147 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001148
1149 tmp = readw(mmio + NV_ADMA_CTL);
1150 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001151 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001152 udelay(1);
1153 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001154 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001155
1156 return 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001157}
1158
1159static void nv_adma_port_stop(struct ata_port *ap)
1160{
Robert Hancockfbbb2622006-10-27 19:08:41 -07001161 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001162 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001163
1164 VPRINTK("ENTER\n");
Robert Hancockfbbb2622006-10-27 19:08:41 -07001165 writew(0, mmio + NV_ADMA_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001166}
1167
Tejun Heo438ac6d2007-03-02 17:31:26 +09001168#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001169static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1170{
1171 struct nv_adma_port_priv *pp = ap->private_data;
1172 void __iomem *mmio = pp->ctl_block;
1173
1174 /* Go to register mode - clears GO */
1175 nv_adma_register_mode(ap);
1176
1177 /* clear CPB fetch count */
1178 writew(0, mmio + NV_ADMA_CPB_COUNT);
1179
1180 /* disable interrupt, shut down port */
1181 writew(0, mmio + NV_ADMA_CTL);
1182
1183 return 0;
1184}
1185
1186static int nv_adma_port_resume(struct ata_port *ap)
1187{
1188 struct nv_adma_port_priv *pp = ap->private_data;
1189 void __iomem *mmio = pp->ctl_block;
1190 u16 tmp;
1191
1192 /* set CPB block location */
1193 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001194 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001195
1196 /* clear any outstanding interrupt conditions */
1197 writew(0xffff, mmio + NV_ADMA_STAT);
1198
1199 /* initialize port variables */
1200 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1201
1202 /* clear CPB fetch count */
1203 writew(0, mmio + NV_ADMA_CPB_COUNT);
1204
1205 /* clear GO for register mode, enable interrupt */
1206 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001207 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1208 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001209
1210 tmp = readw(mmio + NV_ADMA_CTL);
1211 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001212 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001213 udelay(1);
1214 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001215 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001216
1217 return 0;
1218}
Tejun Heo438ac6d2007-03-02 17:31:26 +09001219#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -07001220
Tejun Heo9a829cc2007-04-17 23:44:08 +09001221static void nv_adma_setup_port(struct ata_port *ap)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001222{
Tejun Heo9a829cc2007-04-17 23:44:08 +09001223 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1224 struct ata_ioports *ioport = &ap->ioaddr;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001225
1226 VPRINTK("ENTER\n");
1227
Tejun Heo9a829cc2007-04-17 23:44:08 +09001228 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001229
Tejun Heo0d5ff562007-02-01 15:06:36 +09001230 ioport->cmd_addr = mmio;
1231 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001232 ioport->error_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001233 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1234 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1235 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1236 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1237 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1238 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001239 ioport->status_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001240 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001241 ioport->altstatus_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001242 ioport->ctl_addr = mmio + 0x20;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001243}
1244
Tejun Heo9a829cc2007-04-17 23:44:08 +09001245static int nv_adma_host_init(struct ata_host *host)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001246{
Tejun Heo9a829cc2007-04-17 23:44:08 +09001247 struct pci_dev *pdev = to_pci_dev(host->dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001248 unsigned int i;
1249 u32 tmp32;
1250
1251 VPRINTK("ENTER\n");
1252
1253 /* enable ADMA on the ports */
1254 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1255 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1256 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1257 NV_MCP_SATA_CFG_20_PORT1_EN |
1258 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1259
1260 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1261
Tejun Heo9a829cc2007-04-17 23:44:08 +09001262 for (i = 0; i < host->n_ports; i++)
1263 nv_adma_setup_port(host->ports[i]);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001264
Robert Hancockfbbb2622006-10-27 19:08:41 -07001265 return 0;
1266}
1267
1268static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1269 struct scatterlist *sg,
1270 int idx,
1271 struct nv_adma_prd *aprd)
1272{
Robert Hancock41949ed2007-02-19 19:02:27 -06001273 u8 flags = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001274 if (qc->tf.flags & ATA_TFLAG_WRITE)
1275 flags |= NV_APRD_WRITE;
1276 if (idx == qc->n_elem - 1)
1277 flags |= NV_APRD_END;
1278 else if (idx != 4)
1279 flags |= NV_APRD_CONT;
1280
1281 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1282 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
Robert Hancock2dec7552006-11-26 14:20:19 -06001283 aprd->flags = flags;
Robert Hancock41949ed2007-02-19 19:02:27 -06001284 aprd->packet_len = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001285}
1286
1287static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1288{
1289 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001290 struct nv_adma_prd *aprd;
1291 struct scatterlist *sg;
Tejun Heoff2aeb12007-12-05 16:43:11 +09001292 unsigned int si;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001293
1294 VPRINTK("ENTER\n");
1295
Tejun Heoff2aeb12007-12-05 16:43:11 +09001296 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1297 aprd = (si < 5) ? &cpb->aprd[si] :
1298 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1299 nv_adma_fill_aprd(qc, sg, si, aprd);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001300 }
Tejun Heoff2aeb12007-12-05 16:43:11 +09001301 if (si > 5)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001302 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
Robert Hancock41949ed2007-02-19 19:02:27 -06001303 else
1304 cpb->next_aprd = cpu_to_le64(0);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001305}
1306
Robert Hancock382a6652007-02-05 16:26:02 -08001307static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1308{
1309 struct nv_adma_port_priv *pp = qc->ap->private_data;
1310
1311 /* ADMA engine can only be used for non-ATAPI DMA commands,
Robert Hancock3f3debd2007-11-25 16:59:36 -06001312 or interrupt-driven no-data commands. */
Jeff Garzikb4479162007-10-25 20:47:30 -04001313 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
Robert Hancock3f3debd2007-11-25 16:59:36 -06001314 (qc->tf.flags & ATA_TFLAG_POLLING))
Robert Hancock382a6652007-02-05 16:26:02 -08001315 return 1;
1316
Jeff Garzikb4479162007-10-25 20:47:30 -04001317 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
Robert Hancock382a6652007-02-05 16:26:02 -08001318 (qc->tf.protocol == ATA_PROT_NODATA))
1319 return 0;
1320
1321 return 1;
1322}
1323
Robert Hancockfbbb2622006-10-27 19:08:41 -07001324static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1325{
1326 struct nv_adma_port_priv *pp = qc->ap->private_data;
1327 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1328 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
Robert Hancockfbbb2622006-10-27 19:08:41 -07001329 NV_CPB_CTL_IEN;
1330
Robert Hancock382a6652007-02-05 16:26:02 -08001331 if (nv_adma_use_reg_mode(qc)) {
Robert Hancock3f3debd2007-11-25 16:59:36 -06001332 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1333 (qc->flags & ATA_QCFLAG_DMAMAP));
Robert Hancock2dec7552006-11-26 14:20:19 -06001334 nv_adma_register_mode(qc->ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001335 ata_qc_prep(qc);
1336 return;
1337 }
1338
Robert Hancock41949ed2007-02-19 19:02:27 -06001339 cpb->resp_flags = NV_CPB_RESP_DONE;
1340 wmb();
1341 cpb->ctl_flags = 0;
1342 wmb();
Robert Hancockfbbb2622006-10-27 19:08:41 -07001343
1344 cpb->len = 3;
1345 cpb->tag = qc->tag;
1346 cpb->next_cpb_idx = 0;
1347
1348 /* turn on NCQ flags for NCQ commands */
1349 if (qc->tf.protocol == ATA_PROT_NCQ)
1350 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1351
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001352 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1353
Robert Hancockfbbb2622006-10-27 19:08:41 -07001354 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1355
Jeff Garzikb4479162007-10-25 20:47:30 -04001356 if (qc->flags & ATA_QCFLAG_DMAMAP) {
Robert Hancock382a6652007-02-05 16:26:02 -08001357 nv_adma_fill_sg(qc, cpb);
1358 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1359 } else
1360 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001361
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001362 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1363 until we are finished filling in all of the contents */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001364 wmb();
1365 cpb->ctl_flags = ctl_flags;
Robert Hancock41949ed2007-02-19 19:02:27 -06001366 wmb();
1367 cpb->resp_flags = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001368}
1369
1370static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1371{
Robert Hancock2dec7552006-11-26 14:20:19 -06001372 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001373 void __iomem *mmio = pp->ctl_block;
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001374 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001375
1376 VPRINTK("ENTER\n");
1377
Robert Hancock3f3debd2007-11-25 16:59:36 -06001378 /* We can't handle result taskfile with NCQ commands, since
1379 retrieving the taskfile switches us out of ADMA mode and would abort
1380 existing commands. */
1381 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1382 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1383 ata_dev_printk(qc->dev, KERN_ERR,
1384 "NCQ w/ RESULT_TF not allowed\n");
1385 return AC_ERR_SYSTEM;
1386 }
1387
Robert Hancock382a6652007-02-05 16:26:02 -08001388 if (nv_adma_use_reg_mode(qc)) {
Robert Hancockfbbb2622006-10-27 19:08:41 -07001389 /* use ATA register mode */
Robert Hancock382a6652007-02-05 16:26:02 -08001390 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
Robert Hancock3f3debd2007-11-25 16:59:36 -06001391 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1392 (qc->flags & ATA_QCFLAG_DMAMAP));
Robert Hancockfbbb2622006-10-27 19:08:41 -07001393 nv_adma_register_mode(qc->ap);
1394 return ata_qc_issue_prot(qc);
1395 } else
1396 nv_adma_mode(qc->ap);
1397
1398 /* write append register, command tag in lower 8 bits
1399 and (number of cpbs to append -1) in top 8 bits */
1400 wmb();
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001401
Jeff Garzikb4479162007-10-25 20:47:30 -04001402 if (curr_ncq != pp->last_issue_ncq) {
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001403 /* Seems to need some delay before switching between NCQ and
1404 non-NCQ commands, else we get command timeouts and such. */
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001405 udelay(20);
1406 pp->last_issue_ncq = curr_ncq;
1407 }
1408
Robert Hancockfbbb2622006-10-27 19:08:41 -07001409 writew(qc->tag, mmio + NV_ADMA_APPEND);
1410
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001411 DPRINTK("Issued tag %u\n", qc->tag);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001412
1413 return 0;
1414}
1415
David Howells7d12e782006-10-05 14:55:46 +01001416static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417{
Jeff Garzikcca39742006-08-24 03:19:22 -04001418 struct ata_host *host = dev_instance;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419 unsigned int i;
1420 unsigned int handled = 0;
1421 unsigned long flags;
1422
Jeff Garzikcca39742006-08-24 03:19:22 -04001423 spin_lock_irqsave(&host->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424
Jeff Garzikcca39742006-08-24 03:19:22 -04001425 for (i = 0; i < host->n_ports; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426 struct ata_port *ap;
1427
Jeff Garzikcca39742006-08-24 03:19:22 -04001428 ap = host->ports[i];
Tejun Heoc1389502005-08-22 14:59:24 +09001429 if (ap &&
Jeff Garzik029f5462006-04-02 10:30:40 -04001430 !(ap->flags & ATA_FLAG_DISABLED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431 struct ata_queued_cmd *qc;
1432
Tejun Heo9af5c9c2007-08-06 18:36:22 +09001433 qc = ata_qc_from_tag(ap, ap->link.active_tag);
Albert Leee50362e2005-09-27 17:39:50 +08001434 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435 handled += ata_host_intr(ap, qc);
Andrew Chewb8870302006-01-04 19:13:04 -08001436 else
1437 // No request pending? Clear interrupt status
1438 // anyway, in case there's one pending.
1439 ap->ops->check_status(ap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440 }
1441
1442 }
1443
Jeff Garzikcca39742006-08-24 03:19:22 -04001444 spin_unlock_irqrestore(&host->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445
1446 return IRQ_RETVAL(handled);
1447}
1448
Jeff Garzikcca39742006-08-24 03:19:22 -04001449static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
Tejun Heoada364e2006-06-17 15:49:56 +09001450{
1451 int i, handled = 0;
1452
Jeff Garzikcca39742006-08-24 03:19:22 -04001453 for (i = 0; i < host->n_ports; i++) {
1454 struct ata_port *ap = host->ports[i];
Tejun Heoada364e2006-06-17 15:49:56 +09001455
1456 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1457 handled += nv_host_intr(ap, irq_stat);
1458
1459 irq_stat >>= NV_INT_PORT_SHIFT;
1460 }
1461
1462 return IRQ_RETVAL(handled);
1463}
1464
David Howells7d12e782006-10-05 14:55:46 +01001465static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
Tejun Heoada364e2006-06-17 15:49:56 +09001466{
Jeff Garzikcca39742006-08-24 03:19:22 -04001467 struct ata_host *host = dev_instance;
Tejun Heoada364e2006-06-17 15:49:56 +09001468 u8 irq_stat;
1469 irqreturn_t ret;
1470
Jeff Garzikcca39742006-08-24 03:19:22 -04001471 spin_lock(&host->lock);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001472 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
Jeff Garzikcca39742006-08-24 03:19:22 -04001473 ret = nv_do_interrupt(host, irq_stat);
1474 spin_unlock(&host->lock);
Tejun Heoada364e2006-06-17 15:49:56 +09001475
1476 return ret;
1477}
1478
David Howells7d12e782006-10-05 14:55:46 +01001479static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
Tejun Heoada364e2006-06-17 15:49:56 +09001480{
Jeff Garzikcca39742006-08-24 03:19:22 -04001481 struct ata_host *host = dev_instance;
Tejun Heoada364e2006-06-17 15:49:56 +09001482 u8 irq_stat;
1483 irqreturn_t ret;
1484
Jeff Garzikcca39742006-08-24 03:19:22 -04001485 spin_lock(&host->lock);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001486 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
Jeff Garzikcca39742006-08-24 03:19:22 -04001487 ret = nv_do_interrupt(host, irq_stat);
1488 spin_unlock(&host->lock);
Tejun Heoada364e2006-06-17 15:49:56 +09001489
1490 return ret;
1491}
1492
Tejun Heoda3dbb12007-07-16 14:29:40 +09001493static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495 if (sc_reg > SCR_CONTROL)
Tejun Heoda3dbb12007-07-16 14:29:40 +09001496 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497
Tejun Heoda3dbb12007-07-16 14:29:40 +09001498 *val = ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1499 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500}
1501
Tejun Heoda3dbb12007-07-16 14:29:40 +09001502static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 if (sc_reg > SCR_CONTROL)
Tejun Heoda3dbb12007-07-16 14:29:40 +09001505 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506
Tejun Heo0d5ff562007-02-01 15:06:36 +09001507 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
Tejun Heoda3dbb12007-07-16 14:29:40 +09001508 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509}
1510
Tejun Heo39f87582006-06-17 15:49:56 +09001511static void nv_nf2_freeze(struct ata_port *ap)
1512{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001513 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
Tejun Heo39f87582006-06-17 15:49:56 +09001514 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1515 u8 mask;
1516
Tejun Heo0d5ff562007-02-01 15:06:36 +09001517 mask = ioread8(scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001518 mask &= ~(NV_INT_ALL << shift);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001519 iowrite8(mask, scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001520}
1521
1522static void nv_nf2_thaw(struct ata_port *ap)
1523{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001524 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
Tejun Heo39f87582006-06-17 15:49:56 +09001525 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1526 u8 mask;
1527
Tejun Heo0d5ff562007-02-01 15:06:36 +09001528 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
Tejun Heo39f87582006-06-17 15:49:56 +09001529
Tejun Heo0d5ff562007-02-01 15:06:36 +09001530 mask = ioread8(scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001531 mask |= (NV_INT_MASK << shift);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001532 iowrite8(mask, scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001533}
1534
1535static void nv_ck804_freeze(struct ata_port *ap)
1536{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001537 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
Tejun Heo39f87582006-06-17 15:49:56 +09001538 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1539 u8 mask;
1540
1541 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1542 mask &= ~(NV_INT_ALL << shift);
1543 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1544}
1545
1546static void nv_ck804_thaw(struct ata_port *ap)
1547{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001548 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
Tejun Heo39f87582006-06-17 15:49:56 +09001549 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1550 u8 mask;
1551
1552 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1553
1554 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1555 mask |= (NV_INT_MASK << shift);
1556 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1557}
1558
Kuan Luof140f0f2007-10-15 15:16:53 -04001559static void nv_mcp55_freeze(struct ata_port *ap)
1560{
1561 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1562 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1563 u32 mask;
1564
1565 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1566
1567 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1568 mask &= ~(NV_INT_ALL_MCP55 << shift);
1569 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1570 ata_bmdma_freeze(ap);
1571}
1572
1573static void nv_mcp55_thaw(struct ata_port *ap)
1574{
1575 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1576 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1577 u32 mask;
1578
1579 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1580
1581 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1582 mask |= (NV_INT_MASK_MCP55 << shift);
1583 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1584 ata_bmdma_thaw(ap);
1585}
1586
Tejun Heocc0680a2007-08-06 18:36:23 +09001587static int nv_hardreset(struct ata_link *link, unsigned int *class,
Tejun Heod4b2bab2007-02-02 16:50:52 +09001588 unsigned long deadline)
Tejun Heo39f87582006-06-17 15:49:56 +09001589{
1590 unsigned int dummy;
1591
1592 /* SATA hardreset fails to retrieve proper device signature on
1593 * some controllers. Don't classify on hardreset. For more
Fernando Luis Vázquez Cao647c5952007-11-07 16:33:49 +09001594 * info, see http://bugzilla.kernel.org/show_bug.cgi?id=3352
Tejun Heo39f87582006-06-17 15:49:56 +09001595 */
Tejun Heocc0680a2007-08-06 18:36:23 +09001596 return sata_std_hardreset(link, &dummy, deadline);
Tejun Heo39f87582006-06-17 15:49:56 +09001597}
1598
1599static void nv_error_handler(struct ata_port *ap)
1600{
1601 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1602 nv_hardreset, ata_std_postreset);
1603}
1604
Robert Hancockfbbb2622006-10-27 19:08:41 -07001605static void nv_adma_error_handler(struct ata_port *ap)
1606{
1607 struct nv_adma_port_priv *pp = ap->private_data;
Jeff Garzikb4479162007-10-25 20:47:30 -04001608 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001609 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001610 int i;
1611 u16 tmp;
Jeff Garzika84471f2007-02-26 05:51:33 -05001612
Jeff Garzikb4479162007-10-25 20:47:30 -04001613 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
Robert Hancock2cb27852007-02-11 18:34:44 -06001614 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1615 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1616 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1617 u32 status = readw(mmio + NV_ADMA_STAT);
Robert Hancock08af7412007-02-19 19:01:59 -06001618 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1619 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
Robert Hancock2cb27852007-02-11 18:34:44 -06001620
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001621 ata_port_printk(ap, KERN_ERR,
1622 "EH in ADMA mode, notifier 0x%X "
Robert Hancock08af7412007-02-19 19:01:59 -06001623 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1624 "next cpb count 0x%X next cpb idx 0x%x\n",
1625 notifier, notifier_error, gen_ctl, status,
1626 cpb_count, next_cpb_idx);
Robert Hancock2cb27852007-02-11 18:34:44 -06001627
Jeff Garzikb4479162007-10-25 20:47:30 -04001628 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
Robert Hancock2cb27852007-02-11 18:34:44 -06001629 struct nv_adma_cpb *cpb = &pp->cpb[i];
Jeff Garzikb4479162007-10-25 20:47:30 -04001630 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001631 ap->link.sactive & (1 << i))
Robert Hancock2cb27852007-02-11 18:34:44 -06001632 ata_port_printk(ap, KERN_ERR,
1633 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1634 i, cpb->ctl_flags, cpb->resp_flags);
1635 }
1636 }
Robert Hancockfbbb2622006-10-27 19:08:41 -07001637
Robert Hancockfbbb2622006-10-27 19:08:41 -07001638 /* Push us back into port register mode for error handling. */
1639 nv_adma_register_mode(ap);
1640
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001641 /* Mark all of the CPBs as invalid to prevent them from
1642 being executed */
Jeff Garzikb4479162007-10-25 20:47:30 -04001643 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001644 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1645
1646 /* clear CPB fetch count */
1647 writew(0, mmio + NV_ADMA_CPB_COUNT);
1648
1649 /* Reset channel */
1650 tmp = readw(mmio + NV_ADMA_CTL);
1651 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzikb4479162007-10-25 20:47:30 -04001652 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001653 udelay(1);
1654 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzikb4479162007-10-25 20:47:30 -04001655 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001656 }
1657
1658 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1659 nv_hardreset, ata_std_postreset);
1660}
1661
Kuan Luof140f0f2007-10-15 15:16:53 -04001662static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1663{
1664 struct nv_swncq_port_priv *pp = ap->private_data;
1665 struct defer_queue *dq = &pp->defer_queue;
1666
1667 /* queue is full */
1668 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1669 dq->defer_bits |= (1 << qc->tag);
1670 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1671}
1672
1673static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1674{
1675 struct nv_swncq_port_priv *pp = ap->private_data;
1676 struct defer_queue *dq = &pp->defer_queue;
1677 unsigned int tag;
1678
1679 if (dq->head == dq->tail) /* null queue */
1680 return NULL;
1681
1682 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1683 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1684 WARN_ON(!(dq->defer_bits & (1 << tag)));
1685 dq->defer_bits &= ~(1 << tag);
1686
1687 return ata_qc_from_tag(ap, tag);
1688}
1689
1690static void nv_swncq_fis_reinit(struct ata_port *ap)
1691{
1692 struct nv_swncq_port_priv *pp = ap->private_data;
1693
1694 pp->dhfis_bits = 0;
1695 pp->dmafis_bits = 0;
1696 pp->sdbfis_bits = 0;
1697 pp->ncq_flags = 0;
1698}
1699
1700static void nv_swncq_pp_reinit(struct ata_port *ap)
1701{
1702 struct nv_swncq_port_priv *pp = ap->private_data;
1703 struct defer_queue *dq = &pp->defer_queue;
1704
1705 dq->head = 0;
1706 dq->tail = 0;
1707 dq->defer_bits = 0;
1708 pp->qc_active = 0;
1709 pp->last_issue_tag = ATA_TAG_POISON;
1710 nv_swncq_fis_reinit(ap);
1711}
1712
1713static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1714{
1715 struct nv_swncq_port_priv *pp = ap->private_data;
1716
1717 writew(fis, pp->irq_block);
1718}
1719
1720static void __ata_bmdma_stop(struct ata_port *ap)
1721{
1722 struct ata_queued_cmd qc;
1723
1724 qc.ap = ap;
1725 ata_bmdma_stop(&qc);
1726}
1727
1728static void nv_swncq_ncq_stop(struct ata_port *ap)
1729{
1730 struct nv_swncq_port_priv *pp = ap->private_data;
1731 unsigned int i;
1732 u32 sactive;
1733 u32 done_mask;
1734
1735 ata_port_printk(ap, KERN_ERR,
1736 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1737 ap->qc_active, ap->link.sactive);
1738 ata_port_printk(ap, KERN_ERR,
1739 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1740 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1741 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1742 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1743
1744 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1745 ap->ops->check_status(ap),
1746 ioread8(ap->ioaddr.error_addr));
1747
1748 sactive = readl(pp->sactive_block);
1749 done_mask = pp->qc_active ^ sactive;
1750
1751 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1752 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1753 u8 err = 0;
1754 if (pp->qc_active & (1 << i))
1755 err = 0;
1756 else if (done_mask & (1 << i))
1757 err = 1;
1758 else
1759 continue;
1760
1761 ata_port_printk(ap, KERN_ERR,
1762 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1763 (pp->dhfis_bits >> i) & 0x1,
1764 (pp->dmafis_bits >> i) & 0x1,
1765 (pp->sdbfis_bits >> i) & 0x1,
1766 (sactive >> i) & 0x1,
1767 (err ? "error! tag doesn't exit" : " "));
1768 }
1769
1770 nv_swncq_pp_reinit(ap);
1771 ap->ops->irq_clear(ap);
1772 __ata_bmdma_stop(ap);
1773 nv_swncq_irq_clear(ap, 0xffff);
1774}
1775
1776static void nv_swncq_error_handler(struct ata_port *ap)
1777{
1778 struct ata_eh_context *ehc = &ap->link.eh_context;
1779
1780 if (ap->link.sactive) {
1781 nv_swncq_ncq_stop(ap);
Tejun Heocf480622008-01-24 00:05:14 +09001782 ehc->i.action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04001783 }
1784
1785 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1786 nv_hardreset, ata_std_postreset);
1787}
1788
1789#ifdef CONFIG_PM
1790static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1791{
1792 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1793 u32 tmp;
1794
1795 /* clear irq */
1796 writel(~0, mmio + NV_INT_STATUS_MCP55);
1797
1798 /* disable irq */
1799 writel(0, mmio + NV_INT_ENABLE_MCP55);
1800
1801 /* disable swncq */
1802 tmp = readl(mmio + NV_CTL_MCP55);
1803 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1804 writel(tmp, mmio + NV_CTL_MCP55);
1805
1806 return 0;
1807}
1808
1809static int nv_swncq_port_resume(struct ata_port *ap)
1810{
1811 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1812 u32 tmp;
1813
1814 /* clear irq */
1815 writel(~0, mmio + NV_INT_STATUS_MCP55);
1816
1817 /* enable irq */
1818 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1819
1820 /* enable swncq */
1821 tmp = readl(mmio + NV_CTL_MCP55);
1822 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1823
1824 return 0;
1825}
1826#endif
1827
1828static void nv_swncq_host_init(struct ata_host *host)
1829{
1830 u32 tmp;
1831 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1832 struct pci_dev *pdev = to_pci_dev(host->dev);
1833 u8 regval;
1834
1835 /* disable ECO 398 */
1836 pci_read_config_byte(pdev, 0x7f, &regval);
1837 regval &= ~(1 << 7);
1838 pci_write_config_byte(pdev, 0x7f, regval);
1839
1840 /* enable swncq */
1841 tmp = readl(mmio + NV_CTL_MCP55);
1842 VPRINTK("HOST_CTL:0x%X\n", tmp);
1843 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1844
1845 /* enable irq intr */
1846 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1847 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1848 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1849
1850 /* clear port irq */
1851 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1852}
1853
1854static int nv_swncq_slave_config(struct scsi_device *sdev)
1855{
1856 struct ata_port *ap = ata_shost_to_port(sdev->host);
1857 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1858 struct ata_device *dev;
1859 int rc;
1860 u8 rev;
1861 u8 check_maxtor = 0;
1862 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1863
1864 rc = ata_scsi_slave_config(sdev);
1865 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1866 /* Not a proper libata device, ignore */
1867 return rc;
1868
1869 dev = &ap->link.device[sdev->id];
1870 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1871 return rc;
1872
1873 /* if MCP51 and Maxtor, then disable ncq */
1874 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1875 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1876 check_maxtor = 1;
1877
1878 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1879 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1880 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1881 pci_read_config_byte(pdev, 0x8, &rev);
1882 if (rev <= 0xa2)
1883 check_maxtor = 1;
1884 }
1885
1886 if (!check_maxtor)
1887 return rc;
1888
1889 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1890
1891 if (strncmp(model_num, "Maxtor", 6) == 0) {
1892 ata_scsi_change_queue_depth(sdev, 1);
1893 ata_dev_printk(dev, KERN_NOTICE,
1894 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1895 }
1896
1897 return rc;
1898}
1899
1900static int nv_swncq_port_start(struct ata_port *ap)
1901{
1902 struct device *dev = ap->host->dev;
1903 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1904 struct nv_swncq_port_priv *pp;
1905 int rc;
1906
1907 rc = ata_port_start(ap);
1908 if (rc)
1909 return rc;
1910
1911 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1912 if (!pp)
1913 return -ENOMEM;
1914
1915 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1916 &pp->prd_dma, GFP_KERNEL);
1917 if (!pp->prd)
1918 return -ENOMEM;
1919 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1920
1921 ap->private_data = pp;
1922 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1923 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1924 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1925
1926 return 0;
1927}
1928
1929static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1930{
1931 if (qc->tf.protocol != ATA_PROT_NCQ) {
1932 ata_qc_prep(qc);
1933 return;
1934 }
1935
1936 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1937 return;
1938
1939 nv_swncq_fill_sg(qc);
1940}
1941
1942static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1943{
1944 struct ata_port *ap = qc->ap;
1945 struct scatterlist *sg;
Kuan Luof140f0f2007-10-15 15:16:53 -04001946 struct nv_swncq_port_priv *pp = ap->private_data;
1947 struct ata_prd *prd;
Tejun Heoff2aeb12007-12-05 16:43:11 +09001948 unsigned int si, idx;
Kuan Luof140f0f2007-10-15 15:16:53 -04001949
1950 prd = pp->prd + ATA_MAX_PRD * qc->tag;
1951
1952 idx = 0;
Tejun Heoff2aeb12007-12-05 16:43:11 +09001953 for_each_sg(qc->sg, sg, qc->n_elem, si) {
Kuan Luof140f0f2007-10-15 15:16:53 -04001954 u32 addr, offset;
1955 u32 sg_len, len;
1956
1957 addr = (u32)sg_dma_address(sg);
1958 sg_len = sg_dma_len(sg);
1959
1960 while (sg_len) {
1961 offset = addr & 0xffff;
1962 len = sg_len;
1963 if ((offset + sg_len) > 0x10000)
1964 len = 0x10000 - offset;
1965
1966 prd[idx].addr = cpu_to_le32(addr);
1967 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
1968
1969 idx++;
1970 sg_len -= len;
1971 addr += len;
1972 }
1973 }
1974
Tejun Heoff2aeb12007-12-05 16:43:11 +09001975 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
Kuan Luof140f0f2007-10-15 15:16:53 -04001976}
1977
1978static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
1979 struct ata_queued_cmd *qc)
1980{
1981 struct nv_swncq_port_priv *pp = ap->private_data;
1982
1983 if (qc == NULL)
1984 return 0;
1985
1986 DPRINTK("Enter\n");
1987
1988 writel((1 << qc->tag), pp->sactive_block);
1989 pp->last_issue_tag = qc->tag;
1990 pp->dhfis_bits &= ~(1 << qc->tag);
1991 pp->dmafis_bits &= ~(1 << qc->tag);
1992 pp->qc_active |= (0x1 << qc->tag);
1993
1994 ap->ops->tf_load(ap, &qc->tf); /* load tf registers */
1995 ap->ops->exec_command(ap, &qc->tf);
1996
1997 DPRINTK("Issued tag %u\n", qc->tag);
1998
1999 return 0;
2000}
2001
2002static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2003{
2004 struct ata_port *ap = qc->ap;
2005 struct nv_swncq_port_priv *pp = ap->private_data;
2006
2007 if (qc->tf.protocol != ATA_PROT_NCQ)
2008 return ata_qc_issue_prot(qc);
2009
2010 DPRINTK("Enter\n");
2011
2012 if (!pp->qc_active)
2013 nv_swncq_issue_atacmd(ap, qc);
2014 else
2015 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2016
2017 return 0;
2018}
2019
2020static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2021{
2022 u32 serror;
2023 struct ata_eh_info *ehi = &ap->link.eh_info;
2024
2025 ata_ehi_clear_desc(ehi);
2026
2027 /* AHCI needs SError cleared; otherwise, it might lock up */
2028 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2029 sata_scr_write(&ap->link, SCR_ERROR, serror);
2030
2031 /* analyze @irq_stat */
2032 if (fis & NV_SWNCQ_IRQ_ADDED)
2033 ata_ehi_push_desc(ehi, "hot plug");
2034 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2035 ata_ehi_push_desc(ehi, "hot unplug");
2036
2037 ata_ehi_hotplugged(ehi);
2038
2039 /* okay, let's hand over to EH */
2040 ehi->serror |= serror;
2041
2042 ata_port_freeze(ap);
2043}
2044
2045static int nv_swncq_sdbfis(struct ata_port *ap)
2046{
2047 struct ata_queued_cmd *qc;
2048 struct nv_swncq_port_priv *pp = ap->private_data;
2049 struct ata_eh_info *ehi = &ap->link.eh_info;
2050 u32 sactive;
2051 int nr_done = 0;
2052 u32 done_mask;
2053 int i;
2054 u8 host_stat;
2055 u8 lack_dhfis = 0;
2056
2057 host_stat = ap->ops->bmdma_status(ap);
2058 if (unlikely(host_stat & ATA_DMA_ERR)) {
2059 /* error when transfering data to/from memory */
2060 ata_ehi_clear_desc(ehi);
2061 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2062 ehi->err_mask |= AC_ERR_HOST_BUS;
Tejun Heocf480622008-01-24 00:05:14 +09002063 ehi->action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04002064 return -EINVAL;
2065 }
2066
2067 ap->ops->irq_clear(ap);
2068 __ata_bmdma_stop(ap);
2069
2070 sactive = readl(pp->sactive_block);
2071 done_mask = pp->qc_active ^ sactive;
2072
2073 if (unlikely(done_mask & sactive)) {
2074 ata_ehi_clear_desc(ehi);
2075 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2076 "(%08x->%08x)", pp->qc_active, sactive);
2077 ehi->err_mask |= AC_ERR_HSM;
Tejun Heocf480622008-01-24 00:05:14 +09002078 ehi->action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04002079 return -EINVAL;
2080 }
2081 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2082 if (!(done_mask & (1 << i)))
2083 continue;
2084
2085 qc = ata_qc_from_tag(ap, i);
2086 if (qc) {
2087 ata_qc_complete(qc);
2088 pp->qc_active &= ~(1 << i);
2089 pp->dhfis_bits &= ~(1 << i);
2090 pp->dmafis_bits &= ~(1 << i);
2091 pp->sdbfis_bits |= (1 << i);
2092 nr_done++;
2093 }
2094 }
2095
2096 if (!ap->qc_active) {
2097 DPRINTK("over\n");
2098 nv_swncq_pp_reinit(ap);
2099 return nr_done;
2100 }
2101
2102 if (pp->qc_active & pp->dhfis_bits)
2103 return nr_done;
2104
2105 if ((pp->ncq_flags & ncq_saw_backout) ||
2106 (pp->qc_active ^ pp->dhfis_bits))
2107 /* if the controller cann't get a device to host register FIS,
2108 * The driver needs to reissue the new command.
2109 */
2110 lack_dhfis = 1;
2111
2112 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2113 "SWNCQ:qc_active 0x%X defer_bits %X "
2114 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2115 ap->print_id, ap->qc_active, pp->qc_active,
2116 pp->defer_queue.defer_bits, pp->dhfis_bits,
2117 pp->dmafis_bits, pp->last_issue_tag);
2118
2119 nv_swncq_fis_reinit(ap);
2120
2121 if (lack_dhfis) {
2122 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2123 nv_swncq_issue_atacmd(ap, qc);
2124 return nr_done;
2125 }
2126
2127 if (pp->defer_queue.defer_bits) {
2128 /* send deferral queue command */
2129 qc = nv_swncq_qc_from_dq(ap);
2130 WARN_ON(qc == NULL);
2131 nv_swncq_issue_atacmd(ap, qc);
2132 }
2133
2134 return nr_done;
2135}
2136
2137static inline u32 nv_swncq_tag(struct ata_port *ap)
2138{
2139 struct nv_swncq_port_priv *pp = ap->private_data;
2140 u32 tag;
2141
2142 tag = readb(pp->tag_block) >> 2;
2143 return (tag & 0x1f);
2144}
2145
2146static int nv_swncq_dmafis(struct ata_port *ap)
2147{
2148 struct ata_queued_cmd *qc;
2149 unsigned int rw;
2150 u8 dmactl;
2151 u32 tag;
2152 struct nv_swncq_port_priv *pp = ap->private_data;
2153
2154 __ata_bmdma_stop(ap);
2155 tag = nv_swncq_tag(ap);
2156
2157 DPRINTK("dma setup tag 0x%x\n", tag);
2158 qc = ata_qc_from_tag(ap, tag);
2159
2160 if (unlikely(!qc))
2161 return 0;
2162
2163 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2164
2165 /* load PRD table addr. */
2166 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2167 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2168
2169 /* specify data direction, triple-check start bit is clear */
2170 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2171 dmactl &= ~ATA_DMA_WR;
2172 if (!rw)
2173 dmactl |= ATA_DMA_WR;
2174
2175 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2176
2177 return 1;
2178}
2179
2180static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2181{
2182 struct nv_swncq_port_priv *pp = ap->private_data;
2183 struct ata_queued_cmd *qc;
2184 struct ata_eh_info *ehi = &ap->link.eh_info;
2185 u32 serror;
2186 u8 ata_stat;
2187 int rc = 0;
2188
2189 ata_stat = ap->ops->check_status(ap);
2190 nv_swncq_irq_clear(ap, fis);
2191 if (!fis)
2192 return;
2193
2194 if (ap->pflags & ATA_PFLAG_FROZEN)
2195 return;
2196
2197 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2198 nv_swncq_hotplug(ap, fis);
2199 return;
2200 }
2201
2202 if (!pp->qc_active)
2203 return;
2204
2205 if (ap->ops->scr_read(ap, SCR_ERROR, &serror))
2206 return;
2207 ap->ops->scr_write(ap, SCR_ERROR, serror);
2208
2209 if (ata_stat & ATA_ERR) {
2210 ata_ehi_clear_desc(ehi);
2211 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2212 ehi->err_mask |= AC_ERR_DEV;
2213 ehi->serror |= serror;
Tejun Heocf480622008-01-24 00:05:14 +09002214 ehi->action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04002215 ata_port_freeze(ap);
2216 return;
2217 }
2218
2219 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2220 /* If the IRQ is backout, driver must issue
2221 * the new command again some time later.
2222 */
2223 pp->ncq_flags |= ncq_saw_backout;
2224 }
2225
2226 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2227 pp->ncq_flags |= ncq_saw_sdb;
2228 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2229 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2230 ap->print_id, pp->qc_active, pp->dhfis_bits,
2231 pp->dmafis_bits, readl(pp->sactive_block));
2232 rc = nv_swncq_sdbfis(ap);
2233 if (rc < 0)
2234 goto irq_error;
2235 }
2236
2237 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2238 /* The interrupt indicates the new command
2239 * was transmitted correctly to the drive.
2240 */
2241 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2242 pp->ncq_flags |= ncq_saw_d2h;
2243 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2244 ata_ehi_push_desc(ehi, "illegal fis transaction");
2245 ehi->err_mask |= AC_ERR_HSM;
Tejun Heocf480622008-01-24 00:05:14 +09002246 ehi->action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04002247 goto irq_error;
2248 }
2249
2250 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2251 !(pp->ncq_flags & ncq_saw_dmas)) {
2252 ata_stat = ap->ops->check_status(ap);
2253 if (ata_stat & ATA_BUSY)
2254 goto irq_exit;
2255
2256 if (pp->defer_queue.defer_bits) {
2257 DPRINTK("send next command\n");
2258 qc = nv_swncq_qc_from_dq(ap);
2259 nv_swncq_issue_atacmd(ap, qc);
2260 }
2261 }
2262 }
2263
2264 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2265 /* program the dma controller with appropriate PRD buffers
2266 * and start the DMA transfer for requested command.
2267 */
2268 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2269 pp->ncq_flags |= ncq_saw_dmas;
2270 rc = nv_swncq_dmafis(ap);
2271 }
2272
2273irq_exit:
2274 return;
2275irq_error:
2276 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2277 ata_port_freeze(ap);
2278 return;
2279}
2280
2281static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2282{
2283 struct ata_host *host = dev_instance;
2284 unsigned int i;
2285 unsigned int handled = 0;
2286 unsigned long flags;
2287 u32 irq_stat;
2288
2289 spin_lock_irqsave(&host->lock, flags);
2290
2291 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2292
2293 for (i = 0; i < host->n_ports; i++) {
2294 struct ata_port *ap = host->ports[i];
2295
2296 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2297 if (ap->link.sactive) {
2298 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2299 handled = 1;
2300 } else {
2301 if (irq_stat) /* reserve Hotplug */
2302 nv_swncq_irq_clear(ap, 0xfff0);
2303
2304 handled += nv_host_intr(ap, (u8)irq_stat);
2305 }
2306 }
2307 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2308 }
2309
2310 spin_unlock_irqrestore(&host->lock, flags);
2311
2312 return IRQ_RETVAL(handled);
2313}
2314
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002315static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002316{
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002317 static int printed_version;
Tejun Heo1626aeb2007-05-04 12:43:58 +02002318 const struct ata_port_info *ppi[] = { NULL, NULL };
Tejun Heo9a829cc2007-04-17 23:44:08 +09002319 struct ata_host *host;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002320 struct nv_host_priv *hpriv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321 int rc;
2322 u32 bar;
Tejun Heo0d5ff562007-02-01 15:06:36 +09002323 void __iomem *base;
Robert Hancockfbbb2622006-10-27 19:08:41 -07002324 unsigned long type = ent->driver_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002325
2326 // Make sure this is a SATA controller by counting the number of bars
2327 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2328 // it's an IDE controller and we ignore it.
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002329 for (bar = 0; bar < 6; bar++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002330 if (pci_resource_start(pdev, bar) == 0)
2331 return -ENODEV;
2332
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002333 if (!printed_version++)
Jeff Garzika9524a72005-10-30 14:39:11 -05002334 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002335
Tejun Heo24dc5f32007-01-20 16:00:28 +09002336 rc = pcim_enable_device(pdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002337 if (rc)
Tejun Heo24dc5f32007-01-20 16:00:28 +09002338 return rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002339
Tejun Heo9a829cc2007-04-17 23:44:08 +09002340 /* determine type and allocate host */
Kuan Luof140f0f2007-10-15 15:16:53 -04002341 if (type == CK804 && adma_enabled) {
Robert Hancockfbbb2622006-10-27 19:08:41 -07002342 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2343 type = ADMA;
Robert Hancockfbbb2622006-10-27 19:08:41 -07002344 }
2345
Jeff Garzik360737a2007-10-29 06:49:24 -04002346 if (type == SWNCQ) {
2347 if (swncq_enabled)
2348 dev_printk(KERN_NOTICE, &pdev->dev,
2349 "Using SWNCQ mode\n");
2350 else
2351 type = GENERIC;
2352 }
2353
Tejun Heo1626aeb2007-05-04 12:43:58 +02002354 ppi[0] = &nv_port_info[type];
Tejun Heod583bc12007-07-04 18:02:07 +09002355 rc = ata_pci_prepare_sff_host(pdev, ppi, &host);
Tejun Heo9a829cc2007-04-17 23:44:08 +09002356 if (rc)
2357 return rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002358
Tejun Heo24dc5f32007-01-20 16:00:28 +09002359 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002360 if (!hpriv)
Tejun Heo24dc5f32007-01-20 16:00:28 +09002361 return -ENOMEM;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002362 hpriv->type = type;
Tejun Heo9a829cc2007-04-17 23:44:08 +09002363 host->private_data = hpriv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002364
Tejun Heo9a829cc2007-04-17 23:44:08 +09002365 /* request and iomap NV_MMIO_BAR */
2366 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2367 if (rc)
2368 return rc;
2369
2370 /* configure SCR access */
2371 base = host->iomap[NV_MMIO_BAR];
2372 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2373 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
Jeff Garzik02cbd922006-03-22 23:59:46 -05002374
Tejun Heoada364e2006-06-17 15:49:56 +09002375 /* enable SATA space for CK804 */
Robert Hancockfbbb2622006-10-27 19:08:41 -07002376 if (type >= CK804) {
Tejun Heoada364e2006-06-17 15:49:56 +09002377 u8 regval;
2378
2379 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2380 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2381 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2382 }
2383
Tejun Heo9a829cc2007-04-17 23:44:08 +09002384 /* init ADMA */
Robert Hancockfbbb2622006-10-27 19:08:41 -07002385 if (type == ADMA) {
Tejun Heo9a829cc2007-04-17 23:44:08 +09002386 rc = nv_adma_host_init(host);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002387 if (rc)
Tejun Heo24dc5f32007-01-20 16:00:28 +09002388 return rc;
Jeff Garzik360737a2007-10-29 06:49:24 -04002389 } else if (type == SWNCQ)
Kuan Luof140f0f2007-10-15 15:16:53 -04002390 nv_swncq_host_init(host);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002391
Tejun Heo9a829cc2007-04-17 23:44:08 +09002392 pci_set_master(pdev);
2393 return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
2394 IRQF_SHARED, ppi[0]->sht);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395}
2396
Tejun Heo438ac6d2007-03-02 17:31:26 +09002397#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002398static int nv_pci_device_resume(struct pci_dev *pdev)
2399{
2400 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2401 struct nv_host_priv *hpriv = host->private_data;
Robert Hancockce053fa2007-02-05 16:26:04 -08002402 int rc;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002403
Robert Hancockce053fa2007-02-05 16:26:04 -08002404 rc = ata_pci_device_do_resume(pdev);
Jeff Garzikb4479162007-10-25 20:47:30 -04002405 if (rc)
Robert Hancockce053fa2007-02-05 16:26:04 -08002406 return rc;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002407
2408 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
Jeff Garzikb4479162007-10-25 20:47:30 -04002409 if (hpriv->type >= CK804) {
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002410 u8 regval;
2411
2412 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2413 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2414 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2415 }
Jeff Garzikb4479162007-10-25 20:47:30 -04002416 if (hpriv->type == ADMA) {
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002417 u32 tmp32;
2418 struct nv_adma_port_priv *pp;
2419 /* enable/disable ADMA on the ports appropriately */
2420 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2421
2422 pp = host->ports[0]->private_data;
Jeff Garzikb4479162007-10-25 20:47:30 -04002423 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002424 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002425 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002426 else
2427 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002428 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002429 pp = host->ports[1]->private_data;
Jeff Garzikb4479162007-10-25 20:47:30 -04002430 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002431 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002432 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002433 else
2434 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002435 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002436
2437 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2438 }
2439 }
2440
2441 ata_host_resume(host);
2442
2443 return 0;
2444}
Tejun Heo438ac6d2007-03-02 17:31:26 +09002445#endif
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002446
Jeff Garzikcca39742006-08-24 03:19:22 -04002447static void nv_ck804_host_stop(struct ata_host *host)
Tejun Heoada364e2006-06-17 15:49:56 +09002448{
Jeff Garzikcca39742006-08-24 03:19:22 -04002449 struct pci_dev *pdev = to_pci_dev(host->dev);
Tejun Heoada364e2006-06-17 15:49:56 +09002450 u8 regval;
2451
2452 /* disable SATA space for CK804 */
2453 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2454 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2455 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
Tejun Heoada364e2006-06-17 15:49:56 +09002456}
2457
Robert Hancockfbbb2622006-10-27 19:08:41 -07002458static void nv_adma_host_stop(struct ata_host *host)
2459{
2460 struct pci_dev *pdev = to_pci_dev(host->dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002461 u32 tmp32;
2462
Robert Hancockfbbb2622006-10-27 19:08:41 -07002463 /* disable ADMA on the ports */
2464 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2465 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2466 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2467 NV_MCP_SATA_CFG_20_PORT1_EN |
2468 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2469
2470 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2471
2472 nv_ck804_host_stop(host);
2473}
2474
Linus Torvalds1da177e2005-04-16 15:20:36 -07002475static int __init nv_init(void)
2476{
Pavel Roskinb7887192006-08-10 18:13:18 +09002477 return pci_register_driver(&nv_pci_driver);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002478}
2479
2480static void __exit nv_exit(void)
2481{
2482 pci_unregister_driver(&nv_pci_driver);
2483}
2484
2485module_init(nv_init);
2486module_exit(nv_exit);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002487module_param_named(adma, adma_enabled, bool, 0444);
2488MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
Kuan Luof140f0f2007-10-15 15:16:53 -04002489module_param_named(swncq, swncq_enabled, bool, 0444);
2490MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: false)");
2491