blob: e2e795e5823671732947df073389dc54d814ed6b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Jeff Garzikaa7e16d2005-08-29 15:12:56 -04008 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
Linus Torvalds1da177e2005-04-16 15:20:36 -070021 *
Jeff Garzikaf36d7f2005-08-28 20:18:39 -040022 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
Robert Hancockfbbb2622006-10-27 19:08:41 -070032 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 */
38
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/kernel.h>
40#include <linux/module.h>
41#include <linux/pci.h>
42#include <linux/init.h>
43#include <linux/blkdev.h>
44#include <linux/delay.h>
45#include <linux/interrupt.h>
Jeff Garzika9524a72005-10-30 14:39:11 -050046#include <linux/device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#include <scsi/scsi_host.h>
Robert Hancockfbbb2622006-10-27 19:08:41 -070048#include <scsi/scsi_device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049#include <linux/libata.h>
50
51#define DRV_NAME "sata_nv"
Robert Hancockcdf56bc2007-01-03 18:13:57 -060052#define DRV_VERSION "3.3"
Robert Hancockfbbb2622006-10-27 19:08:41 -070053
54#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
Jeff Garzik10ad05d2006-03-22 23:50:50 -050056enum {
Tejun Heo0d5ff562007-02-01 15:06:36 +090057 NV_MMIO_BAR = 5,
58
Jeff Garzik10ad05d2006-03-22 23:50:50 -050059 NV_PORTS = 2,
60 NV_PIO_MASK = 0x1f,
61 NV_MWDMA_MASK = 0x07,
62 NV_UDMA_MASK = 0x7f,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
Tejun Heo27e4b272006-06-17 15:49:55 +090066 /* INT_STATUS/ENABLE */
Jeff Garzik10ad05d2006-03-22 23:50:50 -050067 NV_INT_STATUS = 0x10,
Jeff Garzik10ad05d2006-03-22 23:50:50 -050068 NV_INT_ENABLE = 0x11,
Tejun Heo27e4b272006-06-17 15:49:55 +090069 NV_INT_STATUS_CK804 = 0x440,
Jeff Garzik10ad05d2006-03-22 23:50:50 -050070 NV_INT_ENABLE_CK804 = 0x441,
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Tejun Heo27e4b272006-06-17 15:49:55 +090072 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
Tejun Heo39f87582006-06-17 15:49:56 +090080 NV_INT_ALL = 0x0f,
Tejun Heo5a44eff2006-06-17 15:49:56 +090081 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
Tejun Heo39f87582006-06-17 15:49:56 +090083
Tejun Heo27e4b272006-06-17 15:49:55 +090084 /* INT_CONFIG */
Jeff Garzik10ad05d2006-03-22 23:50:50 -050085 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
Jeff Garzik10ad05d2006-03-22 23:50:50 -050088 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
Robert Hancockfbbb2622006-10-27 19:08:41 -070091 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
166 NV_ADMA_STAT_TIMEOUT,
167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
Robert Hancock2dec7552006-11-26 14:20:19 -0600170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
Robert Hancockfbbb2622006-10-27 19:08:41 -0700171
Jeff Garzik10ad05d2006-03-22 23:50:50 -0500172};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173
Robert Hancockfbbb2622006-10-27 19:08:41 -0700174/* ADMA Physical Region Descriptor - one SG segment */
175struct nv_adma_prd {
176 __le64 addr;
177 __le32 len;
178 u8 flags;
179 u8 packet_len;
180 __le16 reserved;
181};
182
183enum nv_adma_regbits {
184 CMDEND = (1 << 15), /* end of command list */
185 WNB = (1 << 14), /* wait-not-BSY */
186 IGN = (1 << 13), /* ignore this entry */
187 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
188 DA2 = (1 << (2 + 8)),
189 DA1 = (1 << (1 + 8)),
190 DA0 = (1 << (0 + 8)),
191};
192
193/* ADMA Command Parameter Block
194 The first 5 SG segments are stored inside the Command Parameter Block itself.
195 If there are more than 5 segments the remainder are stored in a separate
196 memory area indicated by next_aprd. */
197struct nv_adma_cpb {
198 u8 resp_flags; /* 0 */
199 u8 reserved1; /* 1 */
200 u8 ctl_flags; /* 2 */
201 /* len is length of taskfile in 64 bit words */
202 u8 len; /* 3 */
203 u8 tag; /* 4 */
204 u8 next_cpb_idx; /* 5 */
205 __le16 reserved2; /* 6-7 */
206 __le16 tf[12]; /* 8-31 */
207 struct nv_adma_prd aprd[5]; /* 32-111 */
208 __le64 next_aprd; /* 112-119 */
209 __le64 reserved3; /* 120-127 */
210};
211
212
213struct nv_adma_port_priv {
214 struct nv_adma_cpb *cpb;
215 dma_addr_t cpb_dma;
216 struct nv_adma_prd *aprd;
217 dma_addr_t aprd_dma;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600218 void __iomem * ctl_block;
219 void __iomem * gen_block;
220 void __iomem * notifier_clear_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700221 u8 flags;
Robert Hancock5e5c74a2007-02-19 18:42:30 -0600222 int last_issue_ncq;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700223};
224
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600225struct nv_host_priv {
226 unsigned long type;
227};
228
Robert Hancockfbbb2622006-10-27 19:08:41 -0700229#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & ( 1 << (19 + (12 * (PORT)))))
230
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600232static void nv_remove_one (struct pci_dev *pdev);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900233#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600234static int nv_pci_device_resume(struct pci_dev *pdev);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900235#endif
Jeff Garzikcca39742006-08-24 03:19:22 -0400236static void nv_ck804_host_stop(struct ata_host *host);
David Howells7d12e782006-10-05 14:55:46 +0100237static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
238static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
239static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg);
241static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
Tejun Heo39f87582006-06-17 15:49:56 +0900243static void nv_nf2_freeze(struct ata_port *ap);
244static void nv_nf2_thaw(struct ata_port *ap);
245static void nv_ck804_freeze(struct ata_port *ap);
246static void nv_ck804_thaw(struct ata_port *ap);
247static void nv_error_handler(struct ata_port *ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700248static int nv_adma_slave_config(struct scsi_device *sdev);
Robert Hancock2dec7552006-11-26 14:20:19 -0600249static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700250static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
251static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
252static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
253static void nv_adma_irq_clear(struct ata_port *ap);
254static int nv_adma_port_start(struct ata_port *ap);
255static void nv_adma_port_stop(struct ata_port *ap);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900256#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600257static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
258static int nv_adma_port_resume(struct ata_port *ap);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900259#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -0700260static void nv_adma_error_handler(struct ata_port *ap);
261static void nv_adma_host_stop(struct ata_host *host);
Robert Hancockf5ecac22007-02-20 21:49:10 -0600262static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
Robert Hancockf2fb3442007-03-26 21:43:36 -0800263static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
Tejun Heo39f87582006-06-17 15:49:56 +0900264
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265enum nv_host_type
266{
267 GENERIC,
268 NFORCE2,
Tejun Heo27e4b272006-06-17 15:49:55 +0900269 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700270 CK804,
271 ADMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272};
273
Jeff Garzik3b7d6972005-11-10 11:04:11 -0500274static const struct pci_device_id nv_pci_tbl[] = {
Jeff Garzik54bb3a92006-09-27 22:20:11 -0400275 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
276 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
277 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
278 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
279 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
280 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
281 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
282 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), GENERIC },
283 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), GENERIC },
284 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), GENERIC },
285 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), GENERIC },
286 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
287 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
288 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
290 PCI_ANY_ID, PCI_ANY_ID,
291 PCI_CLASS_STORAGE_IDE<<8, 0xffff00, GENERIC },
Daniel Drake541134c2005-07-03 13:44:39 +0100292 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
293 PCI_ANY_ID, PCI_ANY_ID,
294 PCI_CLASS_STORAGE_RAID<<8, 0xffff00, GENERIC },
Jeff Garzik2d2744f2006-09-28 20:21:59 -0400295
296 { } /* terminate list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297};
298
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299static struct pci_driver nv_pci_driver = {
300 .name = DRV_NAME,
301 .id_table = nv_pci_tbl,
302 .probe = nv_init_one,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900303#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600304 .suspend = ata_pci_device_suspend,
305 .resume = nv_pci_device_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900306#endif
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600307 .remove = nv_remove_one,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308};
309
Jeff Garzik193515d2005-11-07 00:59:37 -0500310static struct scsi_host_template nv_sht = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 .module = THIS_MODULE,
312 .name = DRV_NAME,
313 .ioctl = ata_scsi_ioctl,
314 .queuecommand = ata_scsi_queuecmd,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 .can_queue = ATA_DEF_QUEUE,
316 .this_id = ATA_SHT_THIS_ID,
317 .sg_tablesize = LIBATA_MAX_PRD,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
319 .emulated = ATA_SHT_EMULATED,
320 .use_clustering = ATA_SHT_USE_CLUSTERING,
321 .proc_name = DRV_NAME,
322 .dma_boundary = ATA_DMA_BOUNDARY,
323 .slave_configure = ata_scsi_slave_config,
Tejun Heoccf68c32006-05-31 18:28:09 +0900324 .slave_destroy = ata_scsi_slave_destroy,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 .bios_param = ata_std_bios_param,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900326#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600327 .suspend = ata_scsi_device_suspend,
328 .resume = ata_scsi_device_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900329#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330};
331
Robert Hancockfbbb2622006-10-27 19:08:41 -0700332static struct scsi_host_template nv_adma_sht = {
333 .module = THIS_MODULE,
334 .name = DRV_NAME,
335 .ioctl = ata_scsi_ioctl,
336 .queuecommand = ata_scsi_queuecmd,
337 .can_queue = NV_ADMA_MAX_CPBS,
338 .this_id = ATA_SHT_THIS_ID,
339 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700340 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
341 .emulated = ATA_SHT_EMULATED,
342 .use_clustering = ATA_SHT_USE_CLUSTERING,
343 .proc_name = DRV_NAME,
344 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
345 .slave_configure = nv_adma_slave_config,
346 .slave_destroy = ata_scsi_slave_destroy,
347 .bios_param = ata_std_bios_param,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900348#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600349 .suspend = ata_scsi_device_suspend,
350 .resume = ata_scsi_device_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900351#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -0700352};
353
Tejun Heoada364e2006-06-17 15:49:56 +0900354static const struct ata_port_operations nv_generic_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 .port_disable = ata_port_disable,
356 .tf_load = ata_tf_load,
357 .tf_read = ata_tf_read,
358 .exec_command = ata_exec_command,
359 .check_status = ata_check_status,
360 .dev_select = ata_std_dev_select,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 .bmdma_setup = ata_bmdma_setup,
362 .bmdma_start = ata_bmdma_start,
363 .bmdma_stop = ata_bmdma_stop,
364 .bmdma_status = ata_bmdma_status,
365 .qc_prep = ata_qc_prep,
366 .qc_issue = ata_qc_issue_prot,
Tejun Heo39f87582006-06-17 15:49:56 +0900367 .freeze = ata_bmdma_freeze,
368 .thaw = ata_bmdma_thaw,
369 .error_handler = nv_error_handler,
370 .post_internal_cmd = ata_bmdma_post_internal_cmd,
Tejun Heo0d5ff562007-02-01 15:06:36 +0900371 .data_xfer = ata_data_xfer,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 .irq_clear = ata_bmdma_irq_clear,
Akira Iguchi246ce3b2007-01-26 16:27:58 +0900373 .irq_on = ata_irq_on,
374 .irq_ack = ata_irq_ack,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375 .scr_read = nv_scr_read,
376 .scr_write = nv_scr_write,
377 .port_start = ata_port_start,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378};
379
Tejun Heoada364e2006-06-17 15:49:56 +0900380static const struct ata_port_operations nv_nf2_ops = {
381 .port_disable = ata_port_disable,
382 .tf_load = ata_tf_load,
383 .tf_read = ata_tf_read,
384 .exec_command = ata_exec_command,
385 .check_status = ata_check_status,
386 .dev_select = ata_std_dev_select,
Tejun Heoada364e2006-06-17 15:49:56 +0900387 .bmdma_setup = ata_bmdma_setup,
388 .bmdma_start = ata_bmdma_start,
389 .bmdma_stop = ata_bmdma_stop,
390 .bmdma_status = ata_bmdma_status,
391 .qc_prep = ata_qc_prep,
392 .qc_issue = ata_qc_issue_prot,
Tejun Heo39f87582006-06-17 15:49:56 +0900393 .freeze = nv_nf2_freeze,
394 .thaw = nv_nf2_thaw,
395 .error_handler = nv_error_handler,
396 .post_internal_cmd = ata_bmdma_post_internal_cmd,
Tejun Heo0d5ff562007-02-01 15:06:36 +0900397 .data_xfer = ata_data_xfer,
Tejun Heoada364e2006-06-17 15:49:56 +0900398 .irq_clear = ata_bmdma_irq_clear,
Akira Iguchi246ce3b2007-01-26 16:27:58 +0900399 .irq_on = ata_irq_on,
400 .irq_ack = ata_irq_ack,
Tejun Heoada364e2006-06-17 15:49:56 +0900401 .scr_read = nv_scr_read,
402 .scr_write = nv_scr_write,
403 .port_start = ata_port_start,
Tejun Heoada364e2006-06-17 15:49:56 +0900404};
405
406static const struct ata_port_operations nv_ck804_ops = {
407 .port_disable = ata_port_disable,
408 .tf_load = ata_tf_load,
409 .tf_read = ata_tf_read,
410 .exec_command = ata_exec_command,
411 .check_status = ata_check_status,
412 .dev_select = ata_std_dev_select,
Tejun Heoada364e2006-06-17 15:49:56 +0900413 .bmdma_setup = ata_bmdma_setup,
414 .bmdma_start = ata_bmdma_start,
415 .bmdma_stop = ata_bmdma_stop,
416 .bmdma_status = ata_bmdma_status,
417 .qc_prep = ata_qc_prep,
418 .qc_issue = ata_qc_issue_prot,
Tejun Heo39f87582006-06-17 15:49:56 +0900419 .freeze = nv_ck804_freeze,
420 .thaw = nv_ck804_thaw,
421 .error_handler = nv_error_handler,
422 .post_internal_cmd = ata_bmdma_post_internal_cmd,
Tejun Heo0d5ff562007-02-01 15:06:36 +0900423 .data_xfer = ata_data_xfer,
Tejun Heoada364e2006-06-17 15:49:56 +0900424 .irq_clear = ata_bmdma_irq_clear,
Akira Iguchi246ce3b2007-01-26 16:27:58 +0900425 .irq_on = ata_irq_on,
426 .irq_ack = ata_irq_ack,
Tejun Heoada364e2006-06-17 15:49:56 +0900427 .scr_read = nv_scr_read,
428 .scr_write = nv_scr_write,
429 .port_start = ata_port_start,
Tejun Heoada364e2006-06-17 15:49:56 +0900430 .host_stop = nv_ck804_host_stop,
431};
432
Robert Hancockfbbb2622006-10-27 19:08:41 -0700433static const struct ata_port_operations nv_adma_ops = {
434 .port_disable = ata_port_disable,
435 .tf_load = ata_tf_load,
Robert Hancockf2fb3442007-03-26 21:43:36 -0800436 .tf_read = nv_adma_tf_read,
Robert Hancock2dec7552006-11-26 14:20:19 -0600437 .check_atapi_dma = nv_adma_check_atapi_dma,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700438 .exec_command = ata_exec_command,
439 .check_status = ata_check_status,
440 .dev_select = ata_std_dev_select,
Robert Hancockf5ecac22007-02-20 21:49:10 -0600441 .bmdma_setup = ata_bmdma_setup,
442 .bmdma_start = ata_bmdma_start,
443 .bmdma_stop = ata_bmdma_stop,
444 .bmdma_status = ata_bmdma_status,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700445 .qc_prep = nv_adma_qc_prep,
446 .qc_issue = nv_adma_qc_issue,
447 .freeze = nv_ck804_freeze,
448 .thaw = nv_ck804_thaw,
449 .error_handler = nv_adma_error_handler,
Robert Hancockf5ecac22007-02-20 21:49:10 -0600450 .post_internal_cmd = nv_adma_post_internal_cmd,
Tejun Heo0d5ff562007-02-01 15:06:36 +0900451 .data_xfer = ata_data_xfer,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700452 .irq_clear = nv_adma_irq_clear,
Akira Iguchi246ce3b2007-01-26 16:27:58 +0900453 .irq_on = ata_irq_on,
454 .irq_ack = ata_irq_ack,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700455 .scr_read = nv_scr_read,
456 .scr_write = nv_scr_write,
457 .port_start = nv_adma_port_start,
458 .port_stop = nv_adma_port_stop,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900459#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600460 .port_suspend = nv_adma_port_suspend,
461 .port_resume = nv_adma_port_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900462#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -0700463 .host_stop = nv_adma_host_stop,
464};
465
Tejun Heoada364e2006-06-17 15:49:56 +0900466static struct ata_port_info nv_port_info[] = {
467 /* generic */
468 {
469 .sht = &nv_sht,
Tejun Heo722420f2006-09-28 17:49:22 +0900470 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
471 ATA_FLAG_HRST_TO_RESUME,
Tejun Heoada364e2006-06-17 15:49:56 +0900472 .pio_mask = NV_PIO_MASK,
473 .mwdma_mask = NV_MWDMA_MASK,
474 .udma_mask = NV_UDMA_MASK,
475 .port_ops = &nv_generic_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900476 .irq_handler = nv_generic_interrupt,
Tejun Heoada364e2006-06-17 15:49:56 +0900477 },
478 /* nforce2/3 */
479 {
480 .sht = &nv_sht,
Tejun Heo722420f2006-09-28 17:49:22 +0900481 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
482 ATA_FLAG_HRST_TO_RESUME,
Tejun Heoada364e2006-06-17 15:49:56 +0900483 .pio_mask = NV_PIO_MASK,
484 .mwdma_mask = NV_MWDMA_MASK,
485 .udma_mask = NV_UDMA_MASK,
486 .port_ops = &nv_nf2_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900487 .irq_handler = nv_nf2_interrupt,
Tejun Heoada364e2006-06-17 15:49:56 +0900488 },
489 /* ck804 */
490 {
491 .sht = &nv_sht,
Tejun Heo722420f2006-09-28 17:49:22 +0900492 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
493 ATA_FLAG_HRST_TO_RESUME,
Tejun Heoada364e2006-06-17 15:49:56 +0900494 .pio_mask = NV_PIO_MASK,
495 .mwdma_mask = NV_MWDMA_MASK,
496 .udma_mask = NV_UDMA_MASK,
497 .port_ops = &nv_ck804_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900498 .irq_handler = nv_ck804_interrupt,
Tejun Heoada364e2006-06-17 15:49:56 +0900499 },
Robert Hancockfbbb2622006-10-27 19:08:41 -0700500 /* ADMA */
501 {
502 .sht = &nv_adma_sht,
503 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600504 ATA_FLAG_HRST_TO_RESUME |
Robert Hancockfbbb2622006-10-27 19:08:41 -0700505 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
506 .pio_mask = NV_PIO_MASK,
507 .mwdma_mask = NV_MWDMA_MASK,
508 .udma_mask = NV_UDMA_MASK,
509 .port_ops = &nv_adma_ops,
Tejun Heo9a829cc2007-04-17 23:44:08 +0900510 .irq_handler = nv_adma_interrupt,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700511 },
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512};
513
514MODULE_AUTHOR("NVIDIA");
515MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
516MODULE_LICENSE("GPL");
517MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
518MODULE_VERSION(DRV_VERSION);
519
Robert Hancockfbbb2622006-10-27 19:08:41 -0700520static int adma_enabled = 1;
521
Robert Hancock2dec7552006-11-26 14:20:19 -0600522static void nv_adma_register_mode(struct ata_port *ap)
523{
Robert Hancock2dec7552006-11-26 14:20:19 -0600524 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600525 void __iomem *mmio = pp->ctl_block;
Robert Hancocka2cfe812007-02-05 16:26:03 -0800526 u16 tmp, status;
527 int count = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600528
529 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
530 return;
531
Robert Hancocka2cfe812007-02-05 16:26:03 -0800532 status = readw(mmio + NV_ADMA_STAT);
533 while(!(status & NV_ADMA_STAT_IDLE) && count < 20) {
534 ndelay(50);
535 status = readw(mmio + NV_ADMA_STAT);
536 count++;
537 }
538 if(count == 20)
539 ata_port_printk(ap, KERN_WARNING,
540 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
541 status);
542
Robert Hancock2dec7552006-11-26 14:20:19 -0600543 tmp = readw(mmio + NV_ADMA_CTL);
544 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
545
Robert Hancocka2cfe812007-02-05 16:26:03 -0800546 count = 0;
547 status = readw(mmio + NV_ADMA_STAT);
548 while(!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
549 ndelay(50);
550 status = readw(mmio + NV_ADMA_STAT);
551 count++;
552 }
553 if(count == 20)
554 ata_port_printk(ap, KERN_WARNING,
555 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
556 status);
557
Robert Hancock2dec7552006-11-26 14:20:19 -0600558 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
559}
560
561static void nv_adma_mode(struct ata_port *ap)
562{
Robert Hancock2dec7552006-11-26 14:20:19 -0600563 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600564 void __iomem *mmio = pp->ctl_block;
Robert Hancocka2cfe812007-02-05 16:26:03 -0800565 u16 tmp, status;
566 int count = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600567
568 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
569 return;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500570
Robert Hancock2dec7552006-11-26 14:20:19 -0600571 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
572
573 tmp = readw(mmio + NV_ADMA_CTL);
574 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
575
Robert Hancocka2cfe812007-02-05 16:26:03 -0800576 status = readw(mmio + NV_ADMA_STAT);
577 while(((status & NV_ADMA_STAT_LEGACY) ||
578 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
579 ndelay(50);
580 status = readw(mmio + NV_ADMA_STAT);
581 count++;
582 }
583 if(count == 20)
584 ata_port_printk(ap, KERN_WARNING,
585 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
586 status);
587
Robert Hancock2dec7552006-11-26 14:20:19 -0600588 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
589}
590
Robert Hancockfbbb2622006-10-27 19:08:41 -0700591static int nv_adma_slave_config(struct scsi_device *sdev)
592{
593 struct ata_port *ap = ata_shost_to_port(sdev->host);
Robert Hancock2dec7552006-11-26 14:20:19 -0600594 struct nv_adma_port_priv *pp = ap->private_data;
595 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700596 u64 bounce_limit;
597 unsigned long segment_boundary;
598 unsigned short sg_tablesize;
599 int rc;
Robert Hancock2dec7552006-11-26 14:20:19 -0600600 int adma_enable;
601 u32 current_reg, new_reg, config_mask;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700602
603 rc = ata_scsi_slave_config(sdev);
604
605 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
606 /* Not a proper libata device, ignore */
607 return rc;
608
609 if (ap->device[sdev->id].class == ATA_DEV_ATAPI) {
610 /*
611 * NVIDIA reports that ADMA mode does not support ATAPI commands.
612 * Therefore ATAPI commands are sent through the legacy interface.
613 * However, the legacy interface only supports 32-bit DMA.
614 * Restrict DMA parameters as required by the legacy interface
615 * when an ATAPI device is connected.
616 */
617 bounce_limit = ATA_DMA_MASK;
618 segment_boundary = ATA_DMA_BOUNDARY;
619 /* Subtract 1 since an extra entry may be needed for padding, see
620 libata-scsi.c */
621 sg_tablesize = LIBATA_MAX_PRD - 1;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500622
Robert Hancock2dec7552006-11-26 14:20:19 -0600623 /* Since the legacy DMA engine is in use, we need to disable ADMA
624 on the port. */
625 adma_enable = 0;
626 nv_adma_register_mode(ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700627 }
628 else {
629 bounce_limit = *ap->dev->dma_mask;
630 segment_boundary = NV_ADMA_DMA_BOUNDARY;
631 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
Robert Hancock2dec7552006-11-26 14:20:19 -0600632 adma_enable = 1;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700633 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500634
Robert Hancock2dec7552006-11-26 14:20:19 -0600635 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700636
Robert Hancock2dec7552006-11-26 14:20:19 -0600637 if(ap->port_no == 1)
638 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
639 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
640 else
641 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
642 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500643
Robert Hancock2dec7552006-11-26 14:20:19 -0600644 if(adma_enable) {
645 new_reg = current_reg | config_mask;
646 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
647 }
648 else {
649 new_reg = current_reg & ~config_mask;
650 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
651 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500652
Robert Hancock2dec7552006-11-26 14:20:19 -0600653 if(current_reg != new_reg)
654 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500655
Robert Hancockfbbb2622006-10-27 19:08:41 -0700656 blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
657 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
658 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
659 ata_port_printk(ap, KERN_INFO,
660 "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
661 (unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
662 return rc;
663}
664
Robert Hancock2dec7552006-11-26 14:20:19 -0600665static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
666{
667 struct nv_adma_port_priv *pp = qc->ap->private_data;
668 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
669}
670
Robert Hancockf2fb3442007-03-26 21:43:36 -0800671static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
672{
673 /* Since commands where a result TF is requested are not
674 executed in ADMA mode, the only time this function will be called
675 in ADMA mode will be if a command fails. In this case we
676 don't care about going into register mode with ADMA commands
677 pending, as the commands will all shortly be aborted anyway. */
678 nv_adma_register_mode(ap);
679
680 ata_tf_read(ap, tf);
681}
682
Robert Hancock2dec7552006-11-26 14:20:19 -0600683static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
Robert Hancockfbbb2622006-10-27 19:08:41 -0700684{
685 unsigned int idx = 0;
686
Robert Hancockac3d6b82007-02-19 19:02:46 -0600687 if(tf->flags & ATA_TFLAG_ISADDR) {
688 if (tf->flags & ATA_TFLAG_LBA48) {
689 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
690 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
691 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
692 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
693 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
694 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
695 } else
696 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
Jeff Garzika84471f2007-02-26 05:51:33 -0500697
Robert Hancockac3d6b82007-02-19 19:02:46 -0600698 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
699 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
700 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
701 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700702 }
Jeff Garzika84471f2007-02-26 05:51:33 -0500703
Robert Hancockac3d6b82007-02-19 19:02:46 -0600704 if(tf->flags & ATA_TFLAG_DEVICE)
705 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700706
707 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
Jeff Garzika84471f2007-02-26 05:51:33 -0500708
Robert Hancockac3d6b82007-02-19 19:02:46 -0600709 while(idx < 12)
710 cpb[idx++] = cpu_to_le16(IGN);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700711
712 return idx;
713}
714
Robert Hancock5bd28a42007-02-05 16:26:01 -0800715static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
Robert Hancockfbbb2622006-10-27 19:08:41 -0700716{
717 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancock2dec7552006-11-26 14:20:19 -0600718 u8 flags = pp->cpb[cpb_num].resp_flags;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700719
720 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
721
Robert Hancock5bd28a42007-02-05 16:26:01 -0800722 if (unlikely((force_err ||
723 flags & (NV_CPB_RESP_ATA_ERR |
724 NV_CPB_RESP_CMD_ERR |
725 NV_CPB_RESP_CPB_ERR)))) {
726 struct ata_eh_info *ehi = &ap->eh_info;
727 int freeze = 0;
728
729 ata_ehi_clear_desc(ehi);
730 ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x", flags );
731 if (flags & NV_CPB_RESP_ATA_ERR) {
732 ata_ehi_push_desc(ehi, ": ATA error");
733 ehi->err_mask |= AC_ERR_DEV;
734 } else if (flags & NV_CPB_RESP_CMD_ERR) {
735 ata_ehi_push_desc(ehi, ": CMD error");
736 ehi->err_mask |= AC_ERR_DEV;
737 } else if (flags & NV_CPB_RESP_CPB_ERR) {
738 ata_ehi_push_desc(ehi, ": CPB error");
739 ehi->err_mask |= AC_ERR_SYSTEM;
740 freeze = 1;
741 } else {
742 /* notifier error, but no error in CPB flags? */
743 ehi->err_mask |= AC_ERR_OTHER;
744 freeze = 1;
745 }
746 /* Kill all commands. EH will determine what actually failed. */
747 if (freeze)
748 ata_port_freeze(ap);
749 else
750 ata_port_abort(ap);
751 return 1;
752 }
753
Robert Hancockf2fb3442007-03-26 21:43:36 -0800754 if (likely(flags & NV_CPB_RESP_DONE)) {
Robert Hancockfbbb2622006-10-27 19:08:41 -0700755 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
Robert Hancock5bd28a42007-02-05 16:26:01 -0800756 VPRINTK("CPB flags done, flags=0x%x\n", flags);
757 if (likely(qc)) {
Robert Hancockf2fb3442007-03-26 21:43:36 -0800758 DPRINTK("Completing qc from tag %d\n",cpb_num);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700759 ata_qc_complete(qc);
Robert Hancock2a54cf72007-02-21 23:53:03 -0600760 } else {
761 struct ata_eh_info *ehi = &ap->eh_info;
762 /* Notifier bits set without a command may indicate the drive
763 is misbehaving. Raise host state machine violation on this
764 condition. */
765 ata_port_printk(ap, KERN_ERR, "notifier for tag %d with no command?\n",
766 cpb_num);
767 ehi->err_mask |= AC_ERR_HSM;
768 ehi->action |= ATA_EH_SOFTRESET;
769 ata_port_freeze(ap);
770 return 1;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700771 }
772 }
Robert Hancock5bd28a42007-02-05 16:26:01 -0800773 return 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700774}
775
Robert Hancock2dec7552006-11-26 14:20:19 -0600776static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
777{
778 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->active_tag);
Robert Hancock2dec7552006-11-26 14:20:19 -0600779
780 /* freeze if hotplugged */
781 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
782 ata_port_freeze(ap);
783 return 1;
784 }
785
786 /* bail out if not our interrupt */
787 if (!(irq_stat & NV_INT_DEV))
788 return 0;
789
790 /* DEV interrupt w/ no active qc? */
791 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
792 ata_check_status(ap);
793 return 1;
794 }
795
796 /* handle interrupt */
Robert Hancockf740d162007-01-23 20:09:02 -0600797 return ata_host_intr(ap, qc);
Robert Hancock2dec7552006-11-26 14:20:19 -0600798}
799
Robert Hancockfbbb2622006-10-27 19:08:41 -0700800static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
801{
802 struct ata_host *host = dev_instance;
803 int i, handled = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600804 u32 notifier_clears[2];
Robert Hancockfbbb2622006-10-27 19:08:41 -0700805
806 spin_lock(&host->lock);
807
808 for (i = 0; i < host->n_ports; i++) {
809 struct ata_port *ap = host->ports[i];
Robert Hancock2dec7552006-11-26 14:20:19 -0600810 notifier_clears[i] = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700811
812 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
813 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600814 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700815 u16 status;
816 u32 gen_ctl;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700817 u32 notifier, notifier_error;
818
819 /* if in ATA register mode, use standard ata interrupt handler */
820 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
Tejun Heo0d5ff562007-02-01 15:06:36 +0900821 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
Robert Hancock2dec7552006-11-26 14:20:19 -0600822 >> (NV_INT_PORT_SHIFT * i);
Robert Hancockf740d162007-01-23 20:09:02 -0600823 if(ata_tag_valid(ap->active_tag))
824 /** NV_INT_DEV indication seems unreliable at times
825 at least in ADMA mode. Force it on always when a
826 command is active, to prevent losing interrupts. */
827 irq_stat |= NV_INT_DEV;
Robert Hancock2dec7552006-11-26 14:20:19 -0600828 handled += nv_host_intr(ap, irq_stat);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700829 continue;
830 }
831
832 notifier = readl(mmio + NV_ADMA_NOTIFIER);
833 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
Robert Hancock2dec7552006-11-26 14:20:19 -0600834 notifier_clears[i] = notifier | notifier_error;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700835
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600836 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700837
Robert Hancockfbbb2622006-10-27 19:08:41 -0700838 if( !NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
839 !notifier_error)
840 /* Nothing to do */
841 continue;
842
843 status = readw(mmio + NV_ADMA_STAT);
844
845 /* Clear status. Ensure the controller sees the clearing before we start
846 looking at any of the CPB statuses, so that any CPB completions after
847 this point in the handler will raise another interrupt. */
848 writew(status, mmio + NV_ADMA_STAT);
849 readw(mmio + NV_ADMA_STAT); /* flush posted write */
850 rmb();
851
Robert Hancock5bd28a42007-02-05 16:26:01 -0800852 handled++; /* irq handled if we got here */
853
854 /* freeze if hotplugged or controller error */
855 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
856 NV_ADMA_STAT_HOTUNPLUG |
Robert Hancock5278b502007-02-11 18:36:56 -0600857 NV_ADMA_STAT_TIMEOUT |
858 NV_ADMA_STAT_SERROR))) {
Robert Hancock5bd28a42007-02-05 16:26:01 -0800859 struct ata_eh_info *ehi = &ap->eh_info;
860
861 ata_ehi_clear_desc(ehi);
862 ata_ehi_push_desc(ehi, "ADMA status 0x%08x", status );
863 if (status & NV_ADMA_STAT_TIMEOUT) {
864 ehi->err_mask |= AC_ERR_SYSTEM;
865 ata_ehi_push_desc(ehi, ": timeout");
866 } else if (status & NV_ADMA_STAT_HOTPLUG) {
867 ata_ehi_hotplugged(ehi);
868 ata_ehi_push_desc(ehi, ": hotplug");
869 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
870 ata_ehi_hotplugged(ehi);
871 ata_ehi_push_desc(ehi, ": hot unplug");
Robert Hancock5278b502007-02-11 18:36:56 -0600872 } else if (status & NV_ADMA_STAT_SERROR) {
873 /* let libata analyze SError and figure out the cause */
874 ata_ehi_push_desc(ehi, ": SError");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800875 }
Robert Hancockfbbb2622006-10-27 19:08:41 -0700876 ata_port_freeze(ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700877 continue;
878 }
879
Robert Hancock5bd28a42007-02-05 16:26:01 -0800880 if (status & (NV_ADMA_STAT_DONE |
881 NV_ADMA_STAT_CPBERR)) {
Robert Hancock8ba5e4c2007-03-08 18:02:18 -0600882 u32 check_commands;
Robert Hancock721449b2007-02-19 19:03:08 -0600883 int pos, error = 0;
Robert Hancock8ba5e4c2007-03-08 18:02:18 -0600884
885 if(ata_tag_valid(ap->active_tag))
886 check_commands = 1 << ap->active_tag;
887 else
888 check_commands = ap->sactive;
889
Robert Hancockfbbb2622006-10-27 19:08:41 -0700890 /** Check CPBs for completed commands */
Robert Hancock721449b2007-02-19 19:03:08 -0600891 while ((pos = ffs(check_commands)) && !error) {
892 pos--;
893 error = nv_adma_check_cpb(ap, pos,
894 notifier_error & (1 << pos) );
895 check_commands &= ~(1 << pos );
Robert Hancockfbbb2622006-10-27 19:08:41 -0700896 }
897 }
Robert Hancockfbbb2622006-10-27 19:08:41 -0700898 }
899 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500900
Robert Hancock2dec7552006-11-26 14:20:19 -0600901 if(notifier_clears[0] || notifier_clears[1]) {
902 /* Note: Both notifier clear registers must be written
903 if either is set, even if one is zero, according to NVIDIA. */
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600904 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
905 writel(notifier_clears[0], pp->notifier_clear_block);
906 pp = host->ports[1]->private_data;
907 writel(notifier_clears[1], pp->notifier_clear_block);
Robert Hancock2dec7552006-11-26 14:20:19 -0600908 }
Robert Hancockfbbb2622006-10-27 19:08:41 -0700909
910 spin_unlock(&host->lock);
911
912 return IRQ_RETVAL(handled);
913}
914
915static void nv_adma_irq_clear(struct ata_port *ap)
916{
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600917 struct nv_adma_port_priv *pp = ap->private_data;
918 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700919 u16 status = readw(mmio + NV_ADMA_STAT);
920 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
921 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
Tejun Heo0d5ff562007-02-01 15:06:36 +0900922 void __iomem *dma_stat_addr = ap->ioaddr.bmdma_addr + ATA_DMA_STATUS;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700923
924 /* clear ADMA status */
925 writew(status, mmio + NV_ADMA_STAT);
926 writel(notifier | notifier_error,
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600927 pp->notifier_clear_block);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700928
929 /** clear legacy status */
Tejun Heo0d5ff562007-02-01 15:06:36 +0900930 iowrite8(ioread8(dma_stat_addr), dma_stat_addr);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700931}
932
Robert Hancockf5ecac22007-02-20 21:49:10 -0600933static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
Robert Hancockfbbb2622006-10-27 19:08:41 -0700934{
Robert Hancockf5ecac22007-02-20 21:49:10 -0600935 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700936
Robert Hancockf5ecac22007-02-20 21:49:10 -0600937 if(pp->flags & NV_ADMA_PORT_REGISTER_MODE)
938 ata_bmdma_post_internal_cmd(qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700939}
940
941static int nv_adma_port_start(struct ata_port *ap)
942{
943 struct device *dev = ap->host->dev;
944 struct nv_adma_port_priv *pp;
945 int rc;
946 void *mem;
947 dma_addr_t mem_dma;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600948 void __iomem *mmio;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700949 u16 tmp;
950
951 VPRINTK("ENTER\n");
952
953 rc = ata_port_start(ap);
954 if (rc)
955 return rc;
956
Tejun Heo24dc5f32007-01-20 16:00:28 +0900957 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
958 if (!pp)
959 return -ENOMEM;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700960
Tejun Heo0d5ff562007-02-01 15:06:36 +0900961 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600962 ap->port_no * NV_ADMA_PORT_SIZE;
963 pp->ctl_block = mmio;
Tejun Heo0d5ff562007-02-01 15:06:36 +0900964 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600965 pp->notifier_clear_block = pp->gen_block +
966 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
967
Tejun Heo24dc5f32007-01-20 16:00:28 +0900968 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
969 &mem_dma, GFP_KERNEL);
970 if (!mem)
971 return -ENOMEM;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700972 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
973
974 /*
975 * First item in chunk of DMA memory:
976 * 128-byte command parameter block (CPB)
977 * one for each command tag
978 */
979 pp->cpb = mem;
980 pp->cpb_dma = mem_dma;
981
982 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
983 writel((mem_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
984
985 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
986 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
987
988 /*
989 * Second item: block of ADMA_SGTBL_LEN s/g entries
990 */
991 pp->aprd = mem;
992 pp->aprd_dma = mem_dma;
993
994 ap->private_data = pp;
995
996 /* clear any outstanding interrupt conditions */
997 writew(0xffff, mmio + NV_ADMA_STAT);
998
999 /* initialize port variables */
1000 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1001
1002 /* clear CPB fetch count */
1003 writew(0, mmio + NV_ADMA_CPB_COUNT);
1004
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001005 /* clear GO for register mode, enable interrupt */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001006 tmp = readw(mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001007 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1008 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001009
1010 tmp = readw(mmio + NV_ADMA_CTL);
1011 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001012 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001013 udelay(1);
1014 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001015 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001016
1017 return 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001018}
1019
1020static void nv_adma_port_stop(struct ata_port *ap)
1021{
Robert Hancockfbbb2622006-10-27 19:08:41 -07001022 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001023 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001024
1025 VPRINTK("ENTER\n");
Robert Hancockfbbb2622006-10-27 19:08:41 -07001026 writew(0, mmio + NV_ADMA_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001027}
1028
Tejun Heo438ac6d2007-03-02 17:31:26 +09001029#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001030static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1031{
1032 struct nv_adma_port_priv *pp = ap->private_data;
1033 void __iomem *mmio = pp->ctl_block;
1034
1035 /* Go to register mode - clears GO */
1036 nv_adma_register_mode(ap);
1037
1038 /* clear CPB fetch count */
1039 writew(0, mmio + NV_ADMA_CPB_COUNT);
1040
1041 /* disable interrupt, shut down port */
1042 writew(0, mmio + NV_ADMA_CTL);
1043
1044 return 0;
1045}
1046
1047static int nv_adma_port_resume(struct ata_port *ap)
1048{
1049 struct nv_adma_port_priv *pp = ap->private_data;
1050 void __iomem *mmio = pp->ctl_block;
1051 u16 tmp;
1052
1053 /* set CPB block location */
1054 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1055 writel((pp->cpb_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1056
1057 /* clear any outstanding interrupt conditions */
1058 writew(0xffff, mmio + NV_ADMA_STAT);
1059
1060 /* initialize port variables */
1061 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1062
1063 /* clear CPB fetch count */
1064 writew(0, mmio + NV_ADMA_CPB_COUNT);
1065
1066 /* clear GO for register mode, enable interrupt */
1067 tmp = readw(mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001068 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1069 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001070
1071 tmp = readw(mmio + NV_ADMA_CTL);
1072 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001073 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001074 udelay(1);
1075 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001076 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001077
1078 return 0;
1079}
Tejun Heo438ac6d2007-03-02 17:31:26 +09001080#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -07001081
Tejun Heo9a829cc2007-04-17 23:44:08 +09001082static void nv_adma_setup_port(struct ata_port *ap)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001083{
Tejun Heo9a829cc2007-04-17 23:44:08 +09001084 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1085 struct ata_ioports *ioport = &ap->ioaddr;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001086
1087 VPRINTK("ENTER\n");
1088
Tejun Heo9a829cc2007-04-17 23:44:08 +09001089 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001090
Tejun Heo0d5ff562007-02-01 15:06:36 +09001091 ioport->cmd_addr = mmio;
1092 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001093 ioport->error_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001094 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1095 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1096 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1097 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1098 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1099 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001100 ioport->status_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001101 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001102 ioport->altstatus_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001103 ioport->ctl_addr = mmio + 0x20;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001104}
1105
Tejun Heo9a829cc2007-04-17 23:44:08 +09001106static int nv_adma_host_init(struct ata_host *host)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001107{
Tejun Heo9a829cc2007-04-17 23:44:08 +09001108 struct pci_dev *pdev = to_pci_dev(host->dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001109 unsigned int i;
1110 u32 tmp32;
1111
1112 VPRINTK("ENTER\n");
1113
1114 /* enable ADMA on the ports */
1115 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1116 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1117 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1118 NV_MCP_SATA_CFG_20_PORT1_EN |
1119 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1120
1121 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1122
Tejun Heo9a829cc2007-04-17 23:44:08 +09001123 for (i = 0; i < host->n_ports; i++)
1124 nv_adma_setup_port(host->ports[i]);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001125
Robert Hancockfbbb2622006-10-27 19:08:41 -07001126 return 0;
1127}
1128
1129static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1130 struct scatterlist *sg,
1131 int idx,
1132 struct nv_adma_prd *aprd)
1133{
Robert Hancock41949ed2007-02-19 19:02:27 -06001134 u8 flags = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001135 if (qc->tf.flags & ATA_TFLAG_WRITE)
1136 flags |= NV_APRD_WRITE;
1137 if (idx == qc->n_elem - 1)
1138 flags |= NV_APRD_END;
1139 else if (idx != 4)
1140 flags |= NV_APRD_CONT;
1141
1142 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1143 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
Robert Hancock2dec7552006-11-26 14:20:19 -06001144 aprd->flags = flags;
Robert Hancock41949ed2007-02-19 19:02:27 -06001145 aprd->packet_len = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001146}
1147
1148static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1149{
1150 struct nv_adma_port_priv *pp = qc->ap->private_data;
1151 unsigned int idx;
1152 struct nv_adma_prd *aprd;
1153 struct scatterlist *sg;
1154
1155 VPRINTK("ENTER\n");
1156
1157 idx = 0;
1158
1159 ata_for_each_sg(sg, qc) {
1160 aprd = (idx < 5) ? &cpb->aprd[idx] : &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (idx-5)];
1161 nv_adma_fill_aprd(qc, sg, idx, aprd);
1162 idx++;
1163 }
1164 if (idx > 5)
1165 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
Robert Hancock41949ed2007-02-19 19:02:27 -06001166 else
1167 cpb->next_aprd = cpu_to_le64(0);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001168}
1169
Robert Hancock382a6652007-02-05 16:26:02 -08001170static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1171{
1172 struct nv_adma_port_priv *pp = qc->ap->private_data;
1173
1174 /* ADMA engine can only be used for non-ATAPI DMA commands,
Robert Hancockf2fb3442007-03-26 21:43:36 -08001175 or interrupt-driven no-data commands, where a result taskfile
1176 is not required. */
Robert Hancock382a6652007-02-05 16:26:02 -08001177 if((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
Robert Hancockf2fb3442007-03-26 21:43:36 -08001178 (qc->tf.flags & ATA_TFLAG_POLLING) ||
1179 (qc->flags & ATA_QCFLAG_RESULT_TF))
Robert Hancock382a6652007-02-05 16:26:02 -08001180 return 1;
1181
1182 if((qc->flags & ATA_QCFLAG_DMAMAP) ||
1183 (qc->tf.protocol == ATA_PROT_NODATA))
1184 return 0;
1185
1186 return 1;
1187}
1188
Robert Hancockfbbb2622006-10-27 19:08:41 -07001189static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1190{
1191 struct nv_adma_port_priv *pp = qc->ap->private_data;
1192 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1193 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
Robert Hancockfbbb2622006-10-27 19:08:41 -07001194 NV_CPB_CTL_IEN;
1195
Robert Hancock382a6652007-02-05 16:26:02 -08001196 if (nv_adma_use_reg_mode(qc)) {
Robert Hancock2dec7552006-11-26 14:20:19 -06001197 nv_adma_register_mode(qc->ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001198 ata_qc_prep(qc);
1199 return;
1200 }
1201
Robert Hancock41949ed2007-02-19 19:02:27 -06001202 cpb->resp_flags = NV_CPB_RESP_DONE;
1203 wmb();
1204 cpb->ctl_flags = 0;
1205 wmb();
Robert Hancockfbbb2622006-10-27 19:08:41 -07001206
1207 cpb->len = 3;
1208 cpb->tag = qc->tag;
1209 cpb->next_cpb_idx = 0;
1210
1211 /* turn on NCQ flags for NCQ commands */
1212 if (qc->tf.protocol == ATA_PROT_NCQ)
1213 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1214
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001215 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1216
Robert Hancockfbbb2622006-10-27 19:08:41 -07001217 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1218
Robert Hancock382a6652007-02-05 16:26:02 -08001219 if(qc->flags & ATA_QCFLAG_DMAMAP) {
1220 nv_adma_fill_sg(qc, cpb);
1221 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1222 } else
1223 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001224
1225 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID until we are
1226 finished filling in all of the contents */
1227 wmb();
1228 cpb->ctl_flags = ctl_flags;
Robert Hancock41949ed2007-02-19 19:02:27 -06001229 wmb();
1230 cpb->resp_flags = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001231}
1232
1233static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1234{
Robert Hancock2dec7552006-11-26 14:20:19 -06001235 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001236 void __iomem *mmio = pp->ctl_block;
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001237 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001238
1239 VPRINTK("ENTER\n");
1240
Robert Hancock382a6652007-02-05 16:26:02 -08001241 if (nv_adma_use_reg_mode(qc)) {
Robert Hancockfbbb2622006-10-27 19:08:41 -07001242 /* use ATA register mode */
Robert Hancock382a6652007-02-05 16:26:02 -08001243 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001244 nv_adma_register_mode(qc->ap);
1245 return ata_qc_issue_prot(qc);
1246 } else
1247 nv_adma_mode(qc->ap);
1248
1249 /* write append register, command tag in lower 8 bits
1250 and (number of cpbs to append -1) in top 8 bits */
1251 wmb();
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001252
1253 if(curr_ncq != pp->last_issue_ncq) {
1254 /* Seems to need some delay before switching between NCQ and non-NCQ
1255 commands, else we get command timeouts and such. */
1256 udelay(20);
1257 pp->last_issue_ncq = curr_ncq;
1258 }
1259
Robert Hancockfbbb2622006-10-27 19:08:41 -07001260 writew(qc->tag, mmio + NV_ADMA_APPEND);
1261
1262 DPRINTK("Issued tag %u\n",qc->tag);
1263
1264 return 0;
1265}
1266
David Howells7d12e782006-10-05 14:55:46 +01001267static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268{
Jeff Garzikcca39742006-08-24 03:19:22 -04001269 struct ata_host *host = dev_instance;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 unsigned int i;
1271 unsigned int handled = 0;
1272 unsigned long flags;
1273
Jeff Garzikcca39742006-08-24 03:19:22 -04001274 spin_lock_irqsave(&host->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275
Jeff Garzikcca39742006-08-24 03:19:22 -04001276 for (i = 0; i < host->n_ports; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277 struct ata_port *ap;
1278
Jeff Garzikcca39742006-08-24 03:19:22 -04001279 ap = host->ports[i];
Tejun Heoc1389502005-08-22 14:59:24 +09001280 if (ap &&
Jeff Garzik029f5462006-04-02 10:30:40 -04001281 !(ap->flags & ATA_FLAG_DISABLED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 struct ata_queued_cmd *qc;
1283
1284 qc = ata_qc_from_tag(ap, ap->active_tag);
Albert Leee50362e2005-09-27 17:39:50 +08001285 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 handled += ata_host_intr(ap, qc);
Andrew Chewb8870302006-01-04 19:13:04 -08001287 else
1288 // No request pending? Clear interrupt status
1289 // anyway, in case there's one pending.
1290 ap->ops->check_status(ap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 }
1292
1293 }
1294
Jeff Garzikcca39742006-08-24 03:19:22 -04001295 spin_unlock_irqrestore(&host->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296
1297 return IRQ_RETVAL(handled);
1298}
1299
Jeff Garzikcca39742006-08-24 03:19:22 -04001300static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
Tejun Heoada364e2006-06-17 15:49:56 +09001301{
1302 int i, handled = 0;
1303
Jeff Garzikcca39742006-08-24 03:19:22 -04001304 for (i = 0; i < host->n_ports; i++) {
1305 struct ata_port *ap = host->ports[i];
Tejun Heoada364e2006-06-17 15:49:56 +09001306
1307 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1308 handled += nv_host_intr(ap, irq_stat);
1309
1310 irq_stat >>= NV_INT_PORT_SHIFT;
1311 }
1312
1313 return IRQ_RETVAL(handled);
1314}
1315
David Howells7d12e782006-10-05 14:55:46 +01001316static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
Tejun Heoada364e2006-06-17 15:49:56 +09001317{
Jeff Garzikcca39742006-08-24 03:19:22 -04001318 struct ata_host *host = dev_instance;
Tejun Heoada364e2006-06-17 15:49:56 +09001319 u8 irq_stat;
1320 irqreturn_t ret;
1321
Jeff Garzikcca39742006-08-24 03:19:22 -04001322 spin_lock(&host->lock);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001323 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
Jeff Garzikcca39742006-08-24 03:19:22 -04001324 ret = nv_do_interrupt(host, irq_stat);
1325 spin_unlock(&host->lock);
Tejun Heoada364e2006-06-17 15:49:56 +09001326
1327 return ret;
1328}
1329
David Howells7d12e782006-10-05 14:55:46 +01001330static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
Tejun Heoada364e2006-06-17 15:49:56 +09001331{
Jeff Garzikcca39742006-08-24 03:19:22 -04001332 struct ata_host *host = dev_instance;
Tejun Heoada364e2006-06-17 15:49:56 +09001333 u8 irq_stat;
1334 irqreturn_t ret;
1335
Jeff Garzikcca39742006-08-24 03:19:22 -04001336 spin_lock(&host->lock);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001337 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
Jeff Garzikcca39742006-08-24 03:19:22 -04001338 ret = nv_do_interrupt(host, irq_stat);
1339 spin_unlock(&host->lock);
Tejun Heoada364e2006-06-17 15:49:56 +09001340
1341 return ret;
1342}
1343
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg)
1345{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346 if (sc_reg > SCR_CONTROL)
1347 return 0xffffffffU;
1348
Tejun Heo0d5ff562007-02-01 15:06:36 +09001349 return ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350}
1351
1352static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val)
1353{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354 if (sc_reg > SCR_CONTROL)
1355 return;
1356
Tejun Heo0d5ff562007-02-01 15:06:36 +09001357 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358}
1359
Tejun Heo39f87582006-06-17 15:49:56 +09001360static void nv_nf2_freeze(struct ata_port *ap)
1361{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001362 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
Tejun Heo39f87582006-06-17 15:49:56 +09001363 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1364 u8 mask;
1365
Tejun Heo0d5ff562007-02-01 15:06:36 +09001366 mask = ioread8(scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001367 mask &= ~(NV_INT_ALL << shift);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001368 iowrite8(mask, scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001369}
1370
1371static void nv_nf2_thaw(struct ata_port *ap)
1372{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001373 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
Tejun Heo39f87582006-06-17 15:49:56 +09001374 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1375 u8 mask;
1376
Tejun Heo0d5ff562007-02-01 15:06:36 +09001377 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
Tejun Heo39f87582006-06-17 15:49:56 +09001378
Tejun Heo0d5ff562007-02-01 15:06:36 +09001379 mask = ioread8(scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001380 mask |= (NV_INT_MASK << shift);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001381 iowrite8(mask, scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001382}
1383
1384static void nv_ck804_freeze(struct ata_port *ap)
1385{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001386 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
Tejun Heo39f87582006-06-17 15:49:56 +09001387 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1388 u8 mask;
1389
1390 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1391 mask &= ~(NV_INT_ALL << shift);
1392 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1393}
1394
1395static void nv_ck804_thaw(struct ata_port *ap)
1396{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001397 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
Tejun Heo39f87582006-06-17 15:49:56 +09001398 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1399 u8 mask;
1400
1401 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1402
1403 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1404 mask |= (NV_INT_MASK << shift);
1405 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1406}
1407
Tejun Heod4b2bab2007-02-02 16:50:52 +09001408static int nv_hardreset(struct ata_port *ap, unsigned int *class,
1409 unsigned long deadline)
Tejun Heo39f87582006-06-17 15:49:56 +09001410{
1411 unsigned int dummy;
1412
1413 /* SATA hardreset fails to retrieve proper device signature on
1414 * some controllers. Don't classify on hardreset. For more
1415 * info, see http://bugme.osdl.org/show_bug.cgi?id=3352
1416 */
Tejun Heod4b2bab2007-02-02 16:50:52 +09001417 return sata_std_hardreset(ap, &dummy, deadline);
Tejun Heo39f87582006-06-17 15:49:56 +09001418}
1419
1420static void nv_error_handler(struct ata_port *ap)
1421{
1422 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1423 nv_hardreset, ata_std_postreset);
1424}
1425
Robert Hancockfbbb2622006-10-27 19:08:41 -07001426static void nv_adma_error_handler(struct ata_port *ap)
1427{
1428 struct nv_adma_port_priv *pp = ap->private_data;
1429 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001430 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001431 int i;
1432 u16 tmp;
Jeff Garzika84471f2007-02-26 05:51:33 -05001433
Robert Hancock2cb27852007-02-11 18:34:44 -06001434 if(ata_tag_valid(ap->active_tag) || ap->sactive) {
1435 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1436 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1437 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1438 u32 status = readw(mmio + NV_ADMA_STAT);
Robert Hancock08af7412007-02-19 19:01:59 -06001439 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1440 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
Robert Hancock2cb27852007-02-11 18:34:44 -06001441
1442 ata_port_printk(ap, KERN_ERR, "EH in ADMA mode, notifier 0x%X "
Robert Hancock08af7412007-02-19 19:01:59 -06001443 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1444 "next cpb count 0x%X next cpb idx 0x%x\n",
1445 notifier, notifier_error, gen_ctl, status,
1446 cpb_count, next_cpb_idx);
Robert Hancock2cb27852007-02-11 18:34:44 -06001447
1448 for( i=0;i<NV_ADMA_MAX_CPBS;i++) {
1449 struct nv_adma_cpb *cpb = &pp->cpb[i];
1450 if( (ata_tag_valid(ap->active_tag) && i == ap->active_tag) ||
1451 ap->sactive & (1 << i) )
1452 ata_port_printk(ap, KERN_ERR,
1453 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1454 i, cpb->ctl_flags, cpb->resp_flags);
1455 }
1456 }
Robert Hancockfbbb2622006-10-27 19:08:41 -07001457
Robert Hancockfbbb2622006-10-27 19:08:41 -07001458 /* Push us back into port register mode for error handling. */
1459 nv_adma_register_mode(ap);
1460
Robert Hancockfbbb2622006-10-27 19:08:41 -07001461 /* Mark all of the CPBs as invalid to prevent them from being executed */
1462 for( i=0;i<NV_ADMA_MAX_CPBS;i++)
1463 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1464
1465 /* clear CPB fetch count */
1466 writew(0, mmio + NV_ADMA_CPB_COUNT);
1467
1468 /* Reset channel */
1469 tmp = readw(mmio + NV_ADMA_CTL);
1470 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001471 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001472 udelay(1);
1473 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Robert Hancock5ce0cf62007-02-19 19:03:27 -06001474 readw( mmio + NV_ADMA_CTL ); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001475 }
1476
1477 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1478 nv_hardreset, ata_std_postreset);
1479}
1480
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
1482{
1483 static int printed_version = 0;
Tejun Heo9a829cc2007-04-17 23:44:08 +09001484 const struct ata_port_info *ppi[2];
1485 struct ata_host *host;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001486 struct nv_host_priv *hpriv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 int rc;
1488 u32 bar;
Tejun Heo0d5ff562007-02-01 15:06:36 +09001489 void __iomem *base;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001490 unsigned long type = ent->driver_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491
1492 // Make sure this is a SATA controller by counting the number of bars
1493 // (NVIDIA SATA controllers will always have six bars). Otherwise,
1494 // it's an IDE controller and we ignore it.
1495 for (bar=0; bar<6; bar++)
1496 if (pci_resource_start(pdev, bar) == 0)
1497 return -ENODEV;
1498
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001499 if (!printed_version++)
Jeff Garzika9524a72005-10-30 14:39:11 -05001500 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501
Tejun Heo24dc5f32007-01-20 16:00:28 +09001502 rc = pcim_enable_device(pdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503 if (rc)
Tejun Heo24dc5f32007-01-20 16:00:28 +09001504 return rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001505
Tejun Heo9a829cc2007-04-17 23:44:08 +09001506 /* determine type and allocate host */
1507 if (type >= CK804 && adma_enabled) {
Robert Hancockfbbb2622006-10-27 19:08:41 -07001508 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
1509 type = ADMA;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001510 }
1511
Tejun Heo9a829cc2007-04-17 23:44:08 +09001512 ppi[0] = ppi[1] = &nv_port_info[type];
1513 rc = ata_pci_prepare_native_host(pdev, ppi, 2, &host);
1514 if (rc)
1515 return rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516
Tejun Heo24dc5f32007-01-20 16:00:28 +09001517 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001518 if (!hpriv)
Tejun Heo24dc5f32007-01-20 16:00:28 +09001519 return -ENOMEM;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001520 hpriv->type = type;
Tejun Heo9a829cc2007-04-17 23:44:08 +09001521 host->private_data = hpriv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
Tejun Heo9a829cc2007-04-17 23:44:08 +09001523 /* set 64bit dma masks, may fail */
1524 if (type == ADMA) {
1525 if (pci_set_dma_mask(pdev, DMA_64BIT_MASK) == 0)
1526 pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1527 }
1528
1529 /* request and iomap NV_MMIO_BAR */
1530 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
1531 if (rc)
1532 return rc;
1533
1534 /* configure SCR access */
1535 base = host->iomap[NV_MMIO_BAR];
1536 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
1537 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
Jeff Garzik02cbd922006-03-22 23:59:46 -05001538
Tejun Heoada364e2006-06-17 15:49:56 +09001539 /* enable SATA space for CK804 */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001540 if (type >= CK804) {
Tejun Heoada364e2006-06-17 15:49:56 +09001541 u8 regval;
1542
1543 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
1544 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1545 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1546 }
1547
Tejun Heo9a829cc2007-04-17 23:44:08 +09001548 /* init ADMA */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001549 if (type == ADMA) {
Tejun Heo9a829cc2007-04-17 23:44:08 +09001550 rc = nv_adma_host_init(host);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001551 if (rc)
Tejun Heo24dc5f32007-01-20 16:00:28 +09001552 return rc;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001553 }
1554
Tejun Heo9a829cc2007-04-17 23:44:08 +09001555 pci_set_master(pdev);
1556 return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
1557 IRQF_SHARED, ppi[0]->sht);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558}
1559
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001560static void nv_remove_one (struct pci_dev *pdev)
1561{
1562 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1563 struct nv_host_priv *hpriv = host->private_data;
1564
1565 ata_pci_remove_one(pdev);
1566 kfree(hpriv);
1567}
1568
Tejun Heo438ac6d2007-03-02 17:31:26 +09001569#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001570static int nv_pci_device_resume(struct pci_dev *pdev)
1571{
1572 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1573 struct nv_host_priv *hpriv = host->private_data;
Robert Hancockce053fa2007-02-05 16:26:04 -08001574 int rc;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001575
Robert Hancockce053fa2007-02-05 16:26:04 -08001576 rc = ata_pci_device_do_resume(pdev);
1577 if(rc)
1578 return rc;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001579
1580 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1581 if(hpriv->type >= CK804) {
1582 u8 regval;
1583
1584 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
1585 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1586 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1587 }
1588 if(hpriv->type == ADMA) {
1589 u32 tmp32;
1590 struct nv_adma_port_priv *pp;
1591 /* enable/disable ADMA on the ports appropriately */
1592 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1593
1594 pp = host->ports[0]->private_data;
1595 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1596 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1597 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1598 else
1599 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
1600 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1601 pp = host->ports[1]->private_data;
1602 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1603 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
1604 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1605 else
1606 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
1607 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1608
1609 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1610 }
1611 }
1612
1613 ata_host_resume(host);
1614
1615 return 0;
1616}
Tejun Heo438ac6d2007-03-02 17:31:26 +09001617#endif
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001618
Jeff Garzikcca39742006-08-24 03:19:22 -04001619static void nv_ck804_host_stop(struct ata_host *host)
Tejun Heoada364e2006-06-17 15:49:56 +09001620{
Jeff Garzikcca39742006-08-24 03:19:22 -04001621 struct pci_dev *pdev = to_pci_dev(host->dev);
Tejun Heoada364e2006-06-17 15:49:56 +09001622 u8 regval;
1623
1624 /* disable SATA space for CK804 */
1625 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
1626 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1627 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
Tejun Heoada364e2006-06-17 15:49:56 +09001628}
1629
Robert Hancockfbbb2622006-10-27 19:08:41 -07001630static void nv_adma_host_stop(struct ata_host *host)
1631{
1632 struct pci_dev *pdev = to_pci_dev(host->dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001633 u32 tmp32;
1634
Robert Hancockfbbb2622006-10-27 19:08:41 -07001635 /* disable ADMA on the ports */
1636 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1637 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1638 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1639 NV_MCP_SATA_CFG_20_PORT1_EN |
1640 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1641
1642 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1643
1644 nv_ck804_host_stop(host);
1645}
1646
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647static int __init nv_init(void)
1648{
Pavel Roskinb7887192006-08-10 18:13:18 +09001649 return pci_register_driver(&nv_pci_driver);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650}
1651
1652static void __exit nv_exit(void)
1653{
1654 pci_unregister_driver(&nv_pci_driver);
1655}
1656
1657module_init(nv_init);
1658module_exit(nv_exit);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001659module_param_named(adma, adma_enabled, bool, 0444);
1660MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");