blob: 55d6179dde589fa0c9a8e66f9c4935137b756acc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 *
Jeff Garzikaa7e16d2005-08-29 15:12:56 -04008 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
Linus Torvalds1da177e2005-04-16 15:20:36 -070021 *
Jeff Garzikaf36d7f2005-08-28 20:18:39 -040022 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
Robert Hancockfbbb2622006-10-27 19:08:41 -070032 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 */
38
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/kernel.h>
40#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090041#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <linux/pci.h>
43#include <linux/init.h>
44#include <linux/blkdev.h>
45#include <linux/delay.h>
46#include <linux/interrupt.h>
Jeff Garzika9524a72005-10-30 14:39:11 -050047#include <linux/device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#include <scsi/scsi_host.h>
Robert Hancockfbbb2622006-10-27 19:08:41 -070049#include <scsi/scsi_device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050#include <linux/libata.h>
51
52#define DRV_NAME "sata_nv"
Jeff Garzik2a3103c2007-08-31 04:54:06 -040053#define DRV_VERSION "3.5"
Robert Hancockfbbb2622006-10-27 19:08:41 -070054
55#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
Jeff Garzik10ad05d2006-03-22 23:50:50 -050057enum {
Tejun Heo0d5ff562007-02-01 15:06:36 +090058 NV_MMIO_BAR = 5,
59
Jeff Garzik10ad05d2006-03-22 23:50:50 -050060 NV_PORTS = 2,
Erik Inge Bolsø14bdef92009-03-14 21:38:24 +010061 NV_PIO_MASK = ATA_PIO4,
62 NV_MWDMA_MASK = ATA_MWDMA2,
63 NV_UDMA_MASK = ATA_UDMA6,
Jeff Garzik10ad05d2006-03-22 23:50:50 -050064 NV_PORT0_SCR_REG_OFFSET = 0x00,
65 NV_PORT1_SCR_REG_OFFSET = 0x40,
Linus Torvalds1da177e2005-04-16 15:20:36 -070066
Tejun Heo27e4b272006-06-17 15:49:55 +090067 /* INT_STATUS/ENABLE */
Jeff Garzik10ad05d2006-03-22 23:50:50 -050068 NV_INT_STATUS = 0x10,
Jeff Garzik10ad05d2006-03-22 23:50:50 -050069 NV_INT_ENABLE = 0x11,
Tejun Heo27e4b272006-06-17 15:49:55 +090070 NV_INT_STATUS_CK804 = 0x440,
Jeff Garzik10ad05d2006-03-22 23:50:50 -050071 NV_INT_ENABLE_CK804 = 0x441,
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
Tejun Heo27e4b272006-06-17 15:49:55 +090073 /* INT_STATUS/ENABLE bits */
74 NV_INT_DEV = 0x01,
75 NV_INT_PM = 0x02,
76 NV_INT_ADDED = 0x04,
77 NV_INT_REMOVED = 0x08,
78
79 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
80
Tejun Heo39f87582006-06-17 15:49:56 +090081 NV_INT_ALL = 0x0f,
Tejun Heo5a44eff2006-06-17 15:49:56 +090082 NV_INT_MASK = NV_INT_DEV |
83 NV_INT_ADDED | NV_INT_REMOVED,
Tejun Heo39f87582006-06-17 15:49:56 +090084
Tejun Heo27e4b272006-06-17 15:49:55 +090085 /* INT_CONFIG */
Jeff Garzik10ad05d2006-03-22 23:50:50 -050086 NV_INT_CONFIG = 0x12,
87 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Jeff Garzik10ad05d2006-03-22 23:50:50 -050089 // For PCI config register 20
90 NV_MCP_SATA_CFG_20 = 0x50,
91 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
Robert Hancockfbbb2622006-10-27 19:08:41 -070092 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
93 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
94 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
95 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
96
97 NV_ADMA_MAX_CPBS = 32,
98 NV_ADMA_CPB_SZ = 128,
99 NV_ADMA_APRD_SZ = 16,
100 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
101 NV_ADMA_APRD_SZ,
102 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
103 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
104 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
105 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
106
107 /* BAR5 offset to ADMA general registers */
108 NV_ADMA_GEN = 0x400,
109 NV_ADMA_GEN_CTL = 0x00,
110 NV_ADMA_NOTIFIER_CLEAR = 0x30,
111
112 /* BAR5 offset to ADMA ports */
113 NV_ADMA_PORT = 0x480,
114
115 /* size of ADMA port register space */
116 NV_ADMA_PORT_SIZE = 0x100,
117
118 /* ADMA port registers */
119 NV_ADMA_CTL = 0x40,
120 NV_ADMA_CPB_COUNT = 0x42,
121 NV_ADMA_NEXT_CPB_IDX = 0x43,
122 NV_ADMA_STAT = 0x44,
123 NV_ADMA_CPB_BASE_LOW = 0x48,
124 NV_ADMA_CPB_BASE_HIGH = 0x4C,
125 NV_ADMA_APPEND = 0x50,
126 NV_ADMA_NOTIFIER = 0x68,
127 NV_ADMA_NOTIFIER_ERROR = 0x6C,
128
129 /* NV_ADMA_CTL register bits */
130 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
131 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
132 NV_ADMA_CTL_GO = (1 << 7),
133 NV_ADMA_CTL_AIEN = (1 << 8),
134 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
135 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
136
137 /* CPB response flag bits */
138 NV_CPB_RESP_DONE = (1 << 0),
139 NV_CPB_RESP_ATA_ERR = (1 << 3),
140 NV_CPB_RESP_CMD_ERR = (1 << 4),
141 NV_CPB_RESP_CPB_ERR = (1 << 7),
142
143 /* CPB control flag bits */
144 NV_CPB_CTL_CPB_VALID = (1 << 0),
145 NV_CPB_CTL_QUEUE = (1 << 1),
146 NV_CPB_CTL_APRD_VALID = (1 << 2),
147 NV_CPB_CTL_IEN = (1 << 3),
148 NV_CPB_CTL_FPDMA = (1 << 4),
149
150 /* APRD flags */
151 NV_APRD_WRITE = (1 << 1),
152 NV_APRD_END = (1 << 2),
153 NV_APRD_CONT = (1 << 3),
154
155 /* NV_ADMA_STAT flags */
156 NV_ADMA_STAT_TIMEOUT = (1 << 0),
157 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
158 NV_ADMA_STAT_HOTPLUG = (1 << 2),
159 NV_ADMA_STAT_CPBERR = (1 << 4),
160 NV_ADMA_STAT_SERROR = (1 << 5),
161 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
162 NV_ADMA_STAT_IDLE = (1 << 8),
163 NV_ADMA_STAT_LEGACY = (1 << 9),
164 NV_ADMA_STAT_STOPPED = (1 << 10),
165 NV_ADMA_STAT_DONE = (1 << 12),
166 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400167 NV_ADMA_STAT_TIMEOUT,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700168
169 /* port flags */
170 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
Robert Hancock2dec7552006-11-26 14:20:19 -0600171 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
Robert Hancockfbbb2622006-10-27 19:08:41 -0700172
Kuan Luof140f0f2007-10-15 15:16:53 -0400173 /* MCP55 reg offset */
174 NV_CTL_MCP55 = 0x400,
175 NV_INT_STATUS_MCP55 = 0x440,
176 NV_INT_ENABLE_MCP55 = 0x444,
177 NV_NCQ_REG_MCP55 = 0x448,
178
179 /* MCP55 */
180 NV_INT_ALL_MCP55 = 0xffff,
181 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
182 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
183
184 /* SWNCQ ENABLE BITS*/
185 NV_CTL_PRI_SWNCQ = 0x02,
186 NV_CTL_SEC_SWNCQ = 0x04,
187
188 /* SW NCQ status bits*/
189 NV_SWNCQ_IRQ_DEV = (1 << 0),
190 NV_SWNCQ_IRQ_PM = (1 << 1),
191 NV_SWNCQ_IRQ_ADDED = (1 << 2),
192 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
193
194 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
195 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
196 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
197 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
198
199 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
200 NV_SWNCQ_IRQ_REMOVED,
201
Jeff Garzik10ad05d2006-03-22 23:50:50 -0500202};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203
Robert Hancockfbbb2622006-10-27 19:08:41 -0700204/* ADMA Physical Region Descriptor - one SG segment */
205struct nv_adma_prd {
206 __le64 addr;
207 __le32 len;
208 u8 flags;
209 u8 packet_len;
210 __le16 reserved;
211};
212
213enum nv_adma_regbits {
214 CMDEND = (1 << 15), /* end of command list */
215 WNB = (1 << 14), /* wait-not-BSY */
216 IGN = (1 << 13), /* ignore this entry */
217 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
218 DA2 = (1 << (2 + 8)),
219 DA1 = (1 << (1 + 8)),
220 DA0 = (1 << (0 + 8)),
221};
222
223/* ADMA Command Parameter Block
224 The first 5 SG segments are stored inside the Command Parameter Block itself.
225 If there are more than 5 segments the remainder are stored in a separate
226 memory area indicated by next_aprd. */
227struct nv_adma_cpb {
228 u8 resp_flags; /* 0 */
229 u8 reserved1; /* 1 */
230 u8 ctl_flags; /* 2 */
231 /* len is length of taskfile in 64 bit words */
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400232 u8 len; /* 3 */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700233 u8 tag; /* 4 */
234 u8 next_cpb_idx; /* 5 */
235 __le16 reserved2; /* 6-7 */
236 __le16 tf[12]; /* 8-31 */
237 struct nv_adma_prd aprd[5]; /* 32-111 */
238 __le64 next_aprd; /* 112-119 */
239 __le64 reserved3; /* 120-127 */
240};
241
242
243struct nv_adma_port_priv {
244 struct nv_adma_cpb *cpb;
245 dma_addr_t cpb_dma;
246 struct nv_adma_prd *aprd;
247 dma_addr_t aprd_dma;
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400248 void __iomem *ctl_block;
249 void __iomem *gen_block;
250 void __iomem *notifier_clear_block;
Robert Hancock8959d302008-02-04 19:39:02 -0600251 u64 adma_dma_mask;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700252 u8 flags;
Robert Hancock5e5c74a2007-02-19 18:42:30 -0600253 int last_issue_ncq;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700254};
255
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600256struct nv_host_priv {
257 unsigned long type;
258};
259
Kuan Luof140f0f2007-10-15 15:16:53 -0400260struct defer_queue {
261 u32 defer_bits;
262 unsigned int head;
263 unsigned int tail;
264 unsigned int tag[ATA_MAX_QUEUE];
265};
266
267enum ncq_saw_flag_list {
268 ncq_saw_d2h = (1U << 0),
269 ncq_saw_dmas = (1U << 1),
270 ncq_saw_sdb = (1U << 2),
271 ncq_saw_backout = (1U << 3),
272};
273
274struct nv_swncq_port_priv {
Tejun Heof60d7012010-05-10 21:41:41 +0200275 struct ata_bmdma_prd *prd; /* our SG list */
Kuan Luof140f0f2007-10-15 15:16:53 -0400276 dma_addr_t prd_dma; /* and its DMA mapping */
277 void __iomem *sactive_block;
278 void __iomem *irq_block;
279 void __iomem *tag_block;
280 u32 qc_active;
281
282 unsigned int last_issue_tag;
283
284 /* fifo circular queue to store deferral command */
285 struct defer_queue defer_queue;
286
287 /* for NCQ interrupt analysis */
288 u32 dhfis_bits;
289 u32 dmafis_bits;
290 u32 sdbfis_bits;
291
292 unsigned int ncq_flags;
293};
294
295
Jeff Garzik5796d1c2007-10-26 00:03:37 -0400296#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
Robert Hancockfbbb2622006-10-27 19:08:41 -0700297
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400298static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900299#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600300static int nv_pci_device_resume(struct pci_dev *pdev);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900301#endif
Jeff Garzikcca39742006-08-24 03:19:22 -0400302static void nv_ck804_host_stop(struct ata_host *host);
David Howells7d12e782006-10-05 14:55:46 +0100303static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
304static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
305static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
Tejun Heo82ef04f2008-07-31 17:02:40 +0900306static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
307static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308
Tejun Heo7f4774b2009-06-10 16:29:07 +0900309static int nv_hardreset(struct ata_link *link, unsigned int *class,
310 unsigned long deadline);
Tejun Heo39f87582006-06-17 15:49:56 +0900311static void nv_nf2_freeze(struct ata_port *ap);
312static void nv_nf2_thaw(struct ata_port *ap);
313static void nv_ck804_freeze(struct ata_port *ap);
314static void nv_ck804_thaw(struct ata_port *ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700315static int nv_adma_slave_config(struct scsi_device *sdev);
Robert Hancock2dec7552006-11-26 14:20:19 -0600316static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700317static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
318static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
319static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
320static void nv_adma_irq_clear(struct ata_port *ap);
321static int nv_adma_port_start(struct ata_port *ap);
322static void nv_adma_port_stop(struct ata_port *ap);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900323#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600324static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
325static int nv_adma_port_resume(struct ata_port *ap);
Tejun Heo438ac6d2007-03-02 17:31:26 +0900326#endif
Robert Hancock53014e22007-05-05 15:36:36 -0600327static void nv_adma_freeze(struct ata_port *ap);
328static void nv_adma_thaw(struct ata_port *ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700329static void nv_adma_error_handler(struct ata_port *ap);
330static void nv_adma_host_stop(struct ata_host *host);
Robert Hancockf5ecac22007-02-20 21:49:10 -0600331static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
Robert Hancockf2fb3442007-03-26 21:43:36 -0800332static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
Tejun Heo39f87582006-06-17 15:49:56 +0900333
Kuan Luof140f0f2007-10-15 15:16:53 -0400334static void nv_mcp55_thaw(struct ata_port *ap);
335static void nv_mcp55_freeze(struct ata_port *ap);
336static void nv_swncq_error_handler(struct ata_port *ap);
337static int nv_swncq_slave_config(struct scsi_device *sdev);
338static int nv_swncq_port_start(struct ata_port *ap);
339static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
340static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
341static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
342static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
343static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
344#ifdef CONFIG_PM
345static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
346static int nv_swncq_port_resume(struct ata_port *ap);
347#endif
348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349enum nv_host_type
350{
351 GENERIC,
352 NFORCE2,
Tejun Heo27e4b272006-06-17 15:49:55 +0900353 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700354 CK804,
Kuan Luof140f0f2007-10-15 15:16:53 -0400355 ADMA,
Tejun Heo2d775702009-01-25 11:29:38 +0900356 MCP5x,
Kuan Luof140f0f2007-10-15 15:16:53 -0400357 SWNCQ,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358};
359
Jeff Garzik3b7d6972005-11-10 11:04:11 -0500360static const struct pci_device_id nv_pci_tbl[] = {
Jeff Garzik54bb3a942006-09-27 22:20:11 -0400361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
Tejun Heo2d775702009-01-25 11:29:38 +0900368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
Kuan Luoe2e031e2007-10-25 02:14:17 -0400372 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
373 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
374 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
Jeff Garzik2d2744f2006-09-28 20:21:59 -0400375
376 { } /* terminate list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377};
378
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379static struct pci_driver nv_pci_driver = {
380 .name = DRV_NAME,
381 .id_table = nv_pci_tbl,
382 .probe = nv_init_one,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900383#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600384 .suspend = ata_pci_device_suspend,
385 .resume = nv_pci_device_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900386#endif
Tejun Heo1daf9ce2007-05-17 13:13:57 +0200387 .remove = ata_pci_remove_one,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388};
389
Jeff Garzik193515d2005-11-07 00:59:37 -0500390static struct scsi_host_template nv_sht = {
Tejun Heo68d1d072008-03-25 12:22:49 +0900391 ATA_BMDMA_SHT(DRV_NAME),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392};
393
Robert Hancockfbbb2622006-10-27 19:08:41 -0700394static struct scsi_host_template nv_adma_sht = {
Tejun Heo68d1d072008-03-25 12:22:49 +0900395 ATA_NCQ_SHT(DRV_NAME),
Robert Hancockfbbb2622006-10-27 19:08:41 -0700396 .can_queue = NV_ADMA_MAX_CPBS,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700397 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700398 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
399 .slave_configure = nv_adma_slave_config,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700400};
401
Kuan Luof140f0f2007-10-15 15:16:53 -0400402static struct scsi_host_template nv_swncq_sht = {
Tejun Heo68d1d072008-03-25 12:22:49 +0900403 ATA_NCQ_SHT(DRV_NAME),
Kuan Luof140f0f2007-10-15 15:16:53 -0400404 .can_queue = ATA_MAX_QUEUE,
Kuan Luof140f0f2007-10-15 15:16:53 -0400405 .sg_tablesize = LIBATA_MAX_PRD,
Kuan Luof140f0f2007-10-15 15:16:53 -0400406 .dma_boundary = ATA_DMA_BOUNDARY,
407 .slave_configure = nv_swncq_slave_config,
Kuan Luof140f0f2007-10-15 15:16:53 -0400408};
409
Tejun Heo7f4774b2009-06-10 16:29:07 +0900410/*
411 * NV SATA controllers have various different problems with hardreset
412 * protocol depending on the specific controller and device.
413 *
414 * GENERIC:
415 *
416 * bko11195 reports that link doesn't come online after hardreset on
417 * generic nv's and there have been several other similar reports on
418 * linux-ide.
419 *
420 * bko12351#c23 reports that warmplug on MCP61 doesn't work with
421 * softreset.
422 *
423 * NF2/3:
424 *
425 * bko3352 reports nf2/3 controllers can't determine device signature
426 * reliably after hardreset. The following thread reports detection
427 * failure on cold boot with the standard debouncing timing.
428 *
429 * http://thread.gmane.org/gmane.linux.ide/34098
430 *
431 * bko12176 reports that hardreset fails to bring up the link during
432 * boot on nf2.
433 *
434 * CK804:
435 *
436 * For initial probing after boot and hot plugging, hardreset mostly
437 * works fine on CK804 but curiously, reprobing on the initial port
438 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
439 * FIS in somewhat undeterministic way.
440 *
441 * SWNCQ:
442 *
443 * bko12351 reports that when SWNCQ is enabled, for hotplug to work,
444 * hardreset should be used and hardreset can't report proper
445 * signature, which suggests that mcp5x is closer to nf2 as long as
446 * reset quirkiness is concerned.
447 *
448 * bko12703 reports that boot probing fails for intel SSD with
449 * hardreset. Link fails to come online. Softreset works fine.
450 *
451 * The failures are varied but the following patterns seem true for
452 * all flavors.
453 *
454 * - Softreset during boot always works.
455 *
456 * - Hardreset during boot sometimes fails to bring up the link on
457 * certain comibnations and device signature acquisition is
458 * unreliable.
459 *
460 * - Hardreset is often necessary after hotplug.
461 *
462 * So, preferring softreset for boot probing and error handling (as
463 * hardreset might bring down the link) but using hardreset for
464 * post-boot probing should work around the above issues in most
465 * cases. Define nv_hardreset() which only kicks in for post-boot
466 * probing and use it for all variants.
467 */
468static struct ata_port_operations nv_generic_ops = {
Tejun Heo029cfd62008-03-25 12:22:49 +0900469 .inherits = &ata_bmdma_port_ops,
Alan Coxc96f1732009-03-24 10:23:46 +0000470 .lost_interrupt = ATA_OP_NULL,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 .scr_read = nv_scr_read,
472 .scr_write = nv_scr_write,
Tejun Heo7f4774b2009-06-10 16:29:07 +0900473 .hardreset = nv_hardreset,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474};
475
Tejun Heo029cfd62008-03-25 12:22:49 +0900476static struct ata_port_operations nv_nf2_ops = {
Tejun Heo7dac7452009-02-12 10:34:32 +0900477 .inherits = &nv_generic_ops,
Tejun Heo39f87582006-06-17 15:49:56 +0900478 .freeze = nv_nf2_freeze,
479 .thaw = nv_nf2_thaw,
Tejun Heoada364e2006-06-17 15:49:56 +0900480};
481
Tejun Heo029cfd62008-03-25 12:22:49 +0900482static struct ata_port_operations nv_ck804_ops = {
Tejun Heo7f4774b2009-06-10 16:29:07 +0900483 .inherits = &nv_generic_ops,
Tejun Heo39f87582006-06-17 15:49:56 +0900484 .freeze = nv_ck804_freeze,
485 .thaw = nv_ck804_thaw,
Tejun Heoada364e2006-06-17 15:49:56 +0900486 .host_stop = nv_ck804_host_stop,
487};
488
Tejun Heo029cfd62008-03-25 12:22:49 +0900489static struct ata_port_operations nv_adma_ops = {
Tejun Heo3c324282008-11-03 12:37:49 +0900490 .inherits = &nv_ck804_ops,
Tejun Heo029cfd62008-03-25 12:22:49 +0900491
Robert Hancock2dec7552006-11-26 14:20:19 -0600492 .check_atapi_dma = nv_adma_check_atapi_dma,
Tejun Heo5682ed32008-04-07 22:47:16 +0900493 .sff_tf_read = nv_adma_tf_read,
Tejun Heo31cc23b2007-09-23 13:14:12 +0900494 .qc_defer = ata_std_qc_defer,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700495 .qc_prep = nv_adma_qc_prep,
496 .qc_issue = nv_adma_qc_issue,
Tejun Heo5682ed32008-04-07 22:47:16 +0900497 .sff_irq_clear = nv_adma_irq_clear,
Tejun Heo029cfd62008-03-25 12:22:49 +0900498
Robert Hancock53014e22007-05-05 15:36:36 -0600499 .freeze = nv_adma_freeze,
500 .thaw = nv_adma_thaw,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700501 .error_handler = nv_adma_error_handler,
Robert Hancockf5ecac22007-02-20 21:49:10 -0600502 .post_internal_cmd = nv_adma_post_internal_cmd,
Tejun Heo029cfd62008-03-25 12:22:49 +0900503
Robert Hancockfbbb2622006-10-27 19:08:41 -0700504 .port_start = nv_adma_port_start,
505 .port_stop = nv_adma_port_stop,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900506#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600507 .port_suspend = nv_adma_port_suspend,
508 .port_resume = nv_adma_port_resume,
Tejun Heo438ac6d2007-03-02 17:31:26 +0900509#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -0700510 .host_stop = nv_adma_host_stop,
511};
512
Tejun Heo029cfd62008-03-25 12:22:49 +0900513static struct ata_port_operations nv_swncq_ops = {
Tejun Heo7f4774b2009-06-10 16:29:07 +0900514 .inherits = &nv_generic_ops,
Tejun Heo029cfd62008-03-25 12:22:49 +0900515
Kuan Luof140f0f2007-10-15 15:16:53 -0400516 .qc_defer = ata_std_qc_defer,
517 .qc_prep = nv_swncq_qc_prep,
518 .qc_issue = nv_swncq_qc_issue,
Tejun Heo029cfd62008-03-25 12:22:49 +0900519
Kuan Luof140f0f2007-10-15 15:16:53 -0400520 .freeze = nv_mcp55_freeze,
521 .thaw = nv_mcp55_thaw,
522 .error_handler = nv_swncq_error_handler,
Tejun Heo029cfd62008-03-25 12:22:49 +0900523
Kuan Luof140f0f2007-10-15 15:16:53 -0400524#ifdef CONFIG_PM
525 .port_suspend = nv_swncq_port_suspend,
526 .port_resume = nv_swncq_port_resume,
527#endif
528 .port_start = nv_swncq_port_start,
529};
530
Tejun Heo95947192008-03-25 12:22:49 +0900531struct nv_pi_priv {
532 irq_handler_t irq_handler;
533 struct scsi_host_template *sht;
534};
535
536#define NV_PI_PRIV(_irq_handler, _sht) \
537 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
538
Tejun Heo1626aeb2007-05-04 12:43:58 +0200539static const struct ata_port_info nv_port_info[] = {
Tejun Heoada364e2006-06-17 15:49:56 +0900540 /* generic */
541 {
Sergei Shtylyov9cbe0562011-02-04 22:05:48 +0300542 .flags = ATA_FLAG_SATA,
Tejun Heoada364e2006-06-17 15:49:56 +0900543 .pio_mask = NV_PIO_MASK,
544 .mwdma_mask = NV_MWDMA_MASK,
545 .udma_mask = NV_UDMA_MASK,
546 .port_ops = &nv_generic_ops,
Tejun Heo95947192008-03-25 12:22:49 +0900547 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
Tejun Heoada364e2006-06-17 15:49:56 +0900548 },
549 /* nforce2/3 */
550 {
Sergei Shtylyov9cbe0562011-02-04 22:05:48 +0300551 .flags = ATA_FLAG_SATA,
Tejun Heoada364e2006-06-17 15:49:56 +0900552 .pio_mask = NV_PIO_MASK,
553 .mwdma_mask = NV_MWDMA_MASK,
554 .udma_mask = NV_UDMA_MASK,
555 .port_ops = &nv_nf2_ops,
Tejun Heo95947192008-03-25 12:22:49 +0900556 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
Tejun Heoada364e2006-06-17 15:49:56 +0900557 },
558 /* ck804 */
559 {
Sergei Shtylyov9cbe0562011-02-04 22:05:48 +0300560 .flags = ATA_FLAG_SATA,
Tejun Heoada364e2006-06-17 15:49:56 +0900561 .pio_mask = NV_PIO_MASK,
562 .mwdma_mask = NV_MWDMA_MASK,
563 .udma_mask = NV_UDMA_MASK,
564 .port_ops = &nv_ck804_ops,
Tejun Heo95947192008-03-25 12:22:49 +0900565 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
Tejun Heoada364e2006-06-17 15:49:56 +0900566 },
Robert Hancockfbbb2622006-10-27 19:08:41 -0700567 /* ADMA */
568 {
Sergei Shtylyov9cbe0562011-02-04 22:05:48 +0300569 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
Robert Hancockfbbb2622006-10-27 19:08:41 -0700570 .pio_mask = NV_PIO_MASK,
571 .mwdma_mask = NV_MWDMA_MASK,
572 .udma_mask = NV_UDMA_MASK,
573 .port_ops = &nv_adma_ops,
Tejun Heo95947192008-03-25 12:22:49 +0900574 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
Robert Hancockfbbb2622006-10-27 19:08:41 -0700575 },
Tejun Heo2d775702009-01-25 11:29:38 +0900576 /* MCP5x */
577 {
Sergei Shtylyov9cbe0562011-02-04 22:05:48 +0300578 .flags = ATA_FLAG_SATA,
Tejun Heo2d775702009-01-25 11:29:38 +0900579 .pio_mask = NV_PIO_MASK,
580 .mwdma_mask = NV_MWDMA_MASK,
581 .udma_mask = NV_UDMA_MASK,
Tejun Heo7f4774b2009-06-10 16:29:07 +0900582 .port_ops = &nv_generic_ops,
Tejun Heo2d775702009-01-25 11:29:38 +0900583 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
584 },
Kuan Luof140f0f2007-10-15 15:16:53 -0400585 /* SWNCQ */
586 {
Sergei Shtylyov9cbe0562011-02-04 22:05:48 +0300587 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
Kuan Luof140f0f2007-10-15 15:16:53 -0400588 .pio_mask = NV_PIO_MASK,
589 .mwdma_mask = NV_MWDMA_MASK,
590 .udma_mask = NV_UDMA_MASK,
591 .port_ops = &nv_swncq_ops,
Tejun Heo95947192008-03-25 12:22:49 +0900592 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
Kuan Luof140f0f2007-10-15 15:16:53 -0400593 },
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594};
595
596MODULE_AUTHOR("NVIDIA");
597MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
598MODULE_LICENSE("GPL");
599MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
600MODULE_VERSION(DRV_VERSION);
601
Rusty Russell90ab5ee2012-01-13 09:32:20 +1030602static bool adma_enabled;
603static bool swncq_enabled = 1;
604static bool msi_enabled;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700605
Robert Hancock2dec7552006-11-26 14:20:19 -0600606static void nv_adma_register_mode(struct ata_port *ap)
607{
Robert Hancock2dec7552006-11-26 14:20:19 -0600608 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600609 void __iomem *mmio = pp->ctl_block;
Robert Hancocka2cfe812007-02-05 16:26:03 -0800610 u16 tmp, status;
611 int count = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600612
613 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
614 return;
615
Robert Hancocka2cfe812007-02-05 16:26:03 -0800616 status = readw(mmio + NV_ADMA_STAT);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400617 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
Robert Hancocka2cfe812007-02-05 16:26:03 -0800618 ndelay(50);
619 status = readw(mmio + NV_ADMA_STAT);
620 count++;
621 }
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400622 if (count == 20)
Joe Perchesa9a79df2011-04-15 15:51:59 -0700623 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n",
624 status);
Robert Hancocka2cfe812007-02-05 16:26:03 -0800625
Robert Hancock2dec7552006-11-26 14:20:19 -0600626 tmp = readw(mmio + NV_ADMA_CTL);
627 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
628
Robert Hancocka2cfe812007-02-05 16:26:03 -0800629 count = 0;
630 status = readw(mmio + NV_ADMA_STAT);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400631 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
Robert Hancocka2cfe812007-02-05 16:26:03 -0800632 ndelay(50);
633 status = readw(mmio + NV_ADMA_STAT);
634 count++;
635 }
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400636 if (count == 20)
Joe Perchesa9a79df2011-04-15 15:51:59 -0700637 ata_port_warn(ap,
638 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
639 status);
Robert Hancocka2cfe812007-02-05 16:26:03 -0800640
Robert Hancock2dec7552006-11-26 14:20:19 -0600641 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
642}
643
644static void nv_adma_mode(struct ata_port *ap)
645{
Robert Hancock2dec7552006-11-26 14:20:19 -0600646 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -0600647 void __iomem *mmio = pp->ctl_block;
Robert Hancocka2cfe812007-02-05 16:26:03 -0800648 u16 tmp, status;
649 int count = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600650
651 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
652 return;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500653
Robert Hancock2dec7552006-11-26 14:20:19 -0600654 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
655
656 tmp = readw(mmio + NV_ADMA_CTL);
657 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
658
Robert Hancocka2cfe812007-02-05 16:26:03 -0800659 status = readw(mmio + NV_ADMA_STAT);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400660 while (((status & NV_ADMA_STAT_LEGACY) ||
Robert Hancocka2cfe812007-02-05 16:26:03 -0800661 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
662 ndelay(50);
663 status = readw(mmio + NV_ADMA_STAT);
664 count++;
665 }
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400666 if (count == 20)
Joe Perchesa9a79df2011-04-15 15:51:59 -0700667 ata_port_warn(ap,
Robert Hancocka2cfe812007-02-05 16:26:03 -0800668 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
669 status);
670
Robert Hancock2dec7552006-11-26 14:20:19 -0600671 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
672}
673
Robert Hancockfbbb2622006-10-27 19:08:41 -0700674static int nv_adma_slave_config(struct scsi_device *sdev)
675{
676 struct ata_port *ap = ata_shost_to_port(sdev->host);
Robert Hancock2dec7552006-11-26 14:20:19 -0600677 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancock8959d302008-02-04 19:39:02 -0600678 struct nv_adma_port_priv *port0, *port1;
679 struct scsi_device *sdev0, *sdev1;
Robert Hancock2dec7552006-11-26 14:20:19 -0600680 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
Robert Hancock8959d302008-02-04 19:39:02 -0600681 unsigned long segment_boundary, flags;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700682 unsigned short sg_tablesize;
683 int rc;
Robert Hancock2dec7552006-11-26 14:20:19 -0600684 int adma_enable;
685 u32 current_reg, new_reg, config_mask;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700686
687 rc = ata_scsi_slave_config(sdev);
688
689 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
690 /* Not a proper libata device, ignore */
691 return rc;
692
Robert Hancock8959d302008-02-04 19:39:02 -0600693 spin_lock_irqsave(ap->lock, flags);
694
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900695 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
Robert Hancockfbbb2622006-10-27 19:08:41 -0700696 /*
697 * NVIDIA reports that ADMA mode does not support ATAPI commands.
698 * Therefore ATAPI commands are sent through the legacy interface.
699 * However, the legacy interface only supports 32-bit DMA.
700 * Restrict DMA parameters as required by the legacy interface
701 * when an ATAPI device is connected.
702 */
Robert Hancockfbbb2622006-10-27 19:08:41 -0700703 segment_boundary = ATA_DMA_BOUNDARY;
704 /* Subtract 1 since an extra entry may be needed for padding, see
705 libata-scsi.c */
706 sg_tablesize = LIBATA_MAX_PRD - 1;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500707
Robert Hancock2dec7552006-11-26 14:20:19 -0600708 /* Since the legacy DMA engine is in use, we need to disable ADMA
709 on the port. */
710 adma_enable = 0;
711 nv_adma_register_mode(ap);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400712 } else {
Robert Hancockfbbb2622006-10-27 19:08:41 -0700713 segment_boundary = NV_ADMA_DMA_BOUNDARY;
714 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
Robert Hancock2dec7552006-11-26 14:20:19 -0600715 adma_enable = 1;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700716 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500717
Robert Hancock2dec7552006-11-26 14:20:19 -0600718 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700719
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400720 if (ap->port_no == 1)
Robert Hancock2dec7552006-11-26 14:20:19 -0600721 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
722 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
723 else
724 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
725 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500726
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400727 if (adma_enable) {
Robert Hancock2dec7552006-11-26 14:20:19 -0600728 new_reg = current_reg | config_mask;
729 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400730 } else {
Robert Hancock2dec7552006-11-26 14:20:19 -0600731 new_reg = current_reg & ~config_mask;
732 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
733 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500734
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400735 if (current_reg != new_reg)
Robert Hancock2dec7552006-11-26 14:20:19 -0600736 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
Jeff Garzikf20b16f2006-12-11 11:14:06 -0500737
Robert Hancock8959d302008-02-04 19:39:02 -0600738 port0 = ap->host->ports[0]->private_data;
739 port1 = ap->host->ports[1]->private_data;
740 sdev0 = ap->host->ports[0]->link.device[0].sdev;
741 sdev1 = ap->host->ports[1]->link.device[0].sdev;
742 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
743 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
744 /** We have to set the DMA mask to 32-bit if either port is in
745 ATAPI mode, since they are on the same PCI device which is
746 used for DMA mapping. If we set the mask we also need to set
747 the bounce limit on both ports to ensure that the block
748 layer doesn't feed addresses that cause DMA mapping to
749 choke. If either SCSI device is not allocated yet, it's OK
750 since that port will discover its correct setting when it
751 does get allocated.
752 Note: Setting 32-bit mask should not fail. */
753 if (sdev0)
754 blk_queue_bounce_limit(sdev0->request_queue,
755 ATA_DMA_MASK);
756 if (sdev1)
757 blk_queue_bounce_limit(sdev1->request_queue,
758 ATA_DMA_MASK);
759
760 pci_set_dma_mask(pdev, ATA_DMA_MASK);
761 } else {
762 /** This shouldn't fail as it was set to this value before */
763 pci_set_dma_mask(pdev, pp->adma_dma_mask);
764 if (sdev0)
765 blk_queue_bounce_limit(sdev0->request_queue,
766 pp->adma_dma_mask);
767 if (sdev1)
768 blk_queue_bounce_limit(sdev1->request_queue,
769 pp->adma_dma_mask);
770 }
771
Robert Hancockfbbb2622006-10-27 19:08:41 -0700772 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
Martin K. Petersen8a783622010-02-26 00:20:39 -0500773 blk_queue_max_segments(sdev->request_queue, sg_tablesize);
Joe Perchesa9a79df2011-04-15 15:51:59 -0700774 ata_port_info(ap,
775 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
776 (unsigned long long)*ap->host->dev->dma_mask,
777 segment_boundary, sg_tablesize);
Robert Hancock8959d302008-02-04 19:39:02 -0600778
779 spin_unlock_irqrestore(ap->lock, flags);
780
Robert Hancockfbbb2622006-10-27 19:08:41 -0700781 return rc;
782}
783
Robert Hancock2dec7552006-11-26 14:20:19 -0600784static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
785{
786 struct nv_adma_port_priv *pp = qc->ap->private_data;
787 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
788}
789
Robert Hancockf2fb3442007-03-26 21:43:36 -0800790static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
791{
Robert Hancock3f3debd2007-11-25 16:59:36 -0600792 /* Other than when internal or pass-through commands are executed,
793 the only time this function will be called in ADMA mode will be
794 if a command fails. In the failure case we don't care about going
795 into register mode with ADMA commands pending, as the commands will
796 all shortly be aborted anyway. We assume that NCQ commands are not
797 issued via passthrough, which is the only way that switching into
798 ADMA mode could abort outstanding commands. */
Robert Hancockf2fb3442007-03-26 21:43:36 -0800799 nv_adma_register_mode(ap);
800
Tejun Heo9363c382008-04-07 22:47:16 +0900801 ata_sff_tf_read(ap, tf);
Robert Hancockf2fb3442007-03-26 21:43:36 -0800802}
803
Robert Hancock2dec7552006-11-26 14:20:19 -0600804static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
Robert Hancockfbbb2622006-10-27 19:08:41 -0700805{
806 unsigned int idx = 0;
807
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400808 if (tf->flags & ATA_TFLAG_ISADDR) {
Robert Hancockac3d6b82007-02-19 19:02:46 -0600809 if (tf->flags & ATA_TFLAG_LBA48) {
810 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
811 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
812 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
813 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
814 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
815 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
816 } else
817 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
Jeff Garzika84471f2007-02-26 05:51:33 -0500818
Robert Hancockac3d6b82007-02-19 19:02:46 -0600819 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
820 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
821 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
822 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700823 }
Jeff Garzika84471f2007-02-26 05:51:33 -0500824
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400825 if (tf->flags & ATA_TFLAG_DEVICE)
Robert Hancockac3d6b82007-02-19 19:02:46 -0600826 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700827
828 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
Jeff Garzika84471f2007-02-26 05:51:33 -0500829
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400830 while (idx < 12)
Robert Hancockac3d6b82007-02-19 19:02:46 -0600831 cpb[idx++] = cpu_to_le16(IGN);
Robert Hancockfbbb2622006-10-27 19:08:41 -0700832
833 return idx;
834}
835
Robert Hancock5bd28a42007-02-05 16:26:01 -0800836static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
Robert Hancockfbbb2622006-10-27 19:08:41 -0700837{
838 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancock2dec7552006-11-26 14:20:19 -0600839 u8 flags = pp->cpb[cpb_num].resp_flags;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700840
841 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
842
Robert Hancock5bd28a42007-02-05 16:26:01 -0800843 if (unlikely((force_err ||
844 flags & (NV_CPB_RESP_ATA_ERR |
845 NV_CPB_RESP_CMD_ERR |
846 NV_CPB_RESP_CPB_ERR)))) {
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900847 struct ata_eh_info *ehi = &ap->link.eh_info;
Robert Hancock5bd28a42007-02-05 16:26:01 -0800848 int freeze = 0;
849
850 ata_ehi_clear_desc(ehi);
Jeff Garzik2dcb4072007-10-19 06:42:56 -0400851 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
Robert Hancock5bd28a42007-02-05 16:26:01 -0800852 if (flags & NV_CPB_RESP_ATA_ERR) {
Tejun Heob64bbc32007-07-16 14:29:39 +0900853 ata_ehi_push_desc(ehi, "ATA error");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800854 ehi->err_mask |= AC_ERR_DEV;
855 } else if (flags & NV_CPB_RESP_CMD_ERR) {
Tejun Heob64bbc32007-07-16 14:29:39 +0900856 ata_ehi_push_desc(ehi, "CMD error");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800857 ehi->err_mask |= AC_ERR_DEV;
858 } else if (flags & NV_CPB_RESP_CPB_ERR) {
Tejun Heob64bbc32007-07-16 14:29:39 +0900859 ata_ehi_push_desc(ehi, "CPB error");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800860 ehi->err_mask |= AC_ERR_SYSTEM;
861 freeze = 1;
862 } else {
863 /* notifier error, but no error in CPB flags? */
Tejun Heob64bbc32007-07-16 14:29:39 +0900864 ata_ehi_push_desc(ehi, "unknown");
Robert Hancock5bd28a42007-02-05 16:26:01 -0800865 ehi->err_mask |= AC_ERR_OTHER;
866 freeze = 1;
867 }
868 /* Kill all commands. EH will determine what actually failed. */
869 if (freeze)
870 ata_port_freeze(ap);
871 else
872 ata_port_abort(ap);
Tejun Heo1aadf5c2010-06-25 15:03:34 +0200873 return -1;
Robert Hancock5bd28a42007-02-05 16:26:01 -0800874 }
875
Tejun Heo1aadf5c2010-06-25 15:03:34 +0200876 if (likely(flags & NV_CPB_RESP_DONE))
877 return 1;
Robert Hancock5bd28a42007-02-05 16:26:01 -0800878 return 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700879}
880
Robert Hancock2dec7552006-11-26 14:20:19 -0600881static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
882{
Tejun Heo9af5c9c2007-08-06 18:36:22 +0900883 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
Robert Hancock2dec7552006-11-26 14:20:19 -0600884
885 /* freeze if hotplugged */
886 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
887 ata_port_freeze(ap);
888 return 1;
889 }
890
891 /* bail out if not our interrupt */
892 if (!(irq_stat & NV_INT_DEV))
893 return 0;
894
895 /* DEV interrupt w/ no active qc? */
896 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
Tejun Heo9363c382008-04-07 22:47:16 +0900897 ata_sff_check_status(ap);
Robert Hancock2dec7552006-11-26 14:20:19 -0600898 return 1;
899 }
900
901 /* handle interrupt */
Tejun Heoc3b28892010-05-19 22:10:21 +0200902 return ata_bmdma_port_intr(ap, qc);
Robert Hancock2dec7552006-11-26 14:20:19 -0600903}
904
Robert Hancockfbbb2622006-10-27 19:08:41 -0700905static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
906{
907 struct ata_host *host = dev_instance;
908 int i, handled = 0;
Robert Hancock2dec7552006-11-26 14:20:19 -0600909 u32 notifier_clears[2];
Robert Hancockfbbb2622006-10-27 19:08:41 -0700910
911 spin_lock(&host->lock);
912
913 for (i = 0; i < host->n_ports; i++) {
914 struct ata_port *ap = host->ports[i];
Tejun Heo3e4ec342010-05-10 21:41:30 +0200915 struct nv_adma_port_priv *pp = ap->private_data;
916 void __iomem *mmio = pp->ctl_block;
917 u16 status;
918 u32 gen_ctl;
919 u32 notifier, notifier_error;
920
Robert Hancock2dec7552006-11-26 14:20:19 -0600921 notifier_clears[i] = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -0700922
Tejun Heo3e4ec342010-05-10 21:41:30 +0200923 /* if ADMA is disabled, use standard ata interrupt handler */
924 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
925 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
926 >> (NV_INT_PORT_SHIFT * i);
927 handled += nv_host_intr(ap, irq_stat);
928 continue;
929 }
Jeff Garzika617c092007-05-21 20:14:23 -0400930
Tejun Heo3e4ec342010-05-10 21:41:30 +0200931 /* if in ATA register mode, check for standard interrupts */
932 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
933 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
934 >> (NV_INT_PORT_SHIFT * i);
935 if (ata_tag_valid(ap->link.active_tag))
936 /** NV_INT_DEV indication seems unreliable
937 at times at least in ADMA mode. Force it
938 on always when a command is active, to
939 prevent losing interrupts. */
940 irq_stat |= NV_INT_DEV;
941 handled += nv_host_intr(ap, irq_stat);
942 }
Robert Hancockfbbb2622006-10-27 19:08:41 -0700943
Tejun Heo3e4ec342010-05-10 21:41:30 +0200944 notifier = readl(mmio + NV_ADMA_NOTIFIER);
945 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
946 notifier_clears[i] = notifier | notifier_error;
947
948 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
949
950 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
951 !notifier_error)
952 /* Nothing to do */
953 continue;
954
955 status = readw(mmio + NV_ADMA_STAT);
956
957 /*
958 * Clear status. Ensure the controller sees the
959 * clearing before we start looking at any of the CPB
960 * statuses, so that any CPB completions after this
961 * point in the handler will raise another interrupt.
962 */
963 writew(status, mmio + NV_ADMA_STAT);
964 readw(mmio + NV_ADMA_STAT); /* flush posted write */
965 rmb();
966
967 handled++; /* irq handled if we got here */
968
969 /* freeze if hotplugged or controller error */
970 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
971 NV_ADMA_STAT_HOTUNPLUG |
972 NV_ADMA_STAT_TIMEOUT |
973 NV_ADMA_STAT_SERROR))) {
974 struct ata_eh_info *ehi = &ap->link.eh_info;
975
976 ata_ehi_clear_desc(ehi);
977 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
978 if (status & NV_ADMA_STAT_TIMEOUT) {
979 ehi->err_mask |= AC_ERR_SYSTEM;
980 ata_ehi_push_desc(ehi, "timeout");
981 } else if (status & NV_ADMA_STAT_HOTPLUG) {
982 ata_ehi_hotplugged(ehi);
983 ata_ehi_push_desc(ehi, "hotplug");
984 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
985 ata_ehi_hotplugged(ehi);
986 ata_ehi_push_desc(ehi, "hot unplug");
987 } else if (status & NV_ADMA_STAT_SERROR) {
988 /* let EH analyze SError and figure out cause */
989 ata_ehi_push_desc(ehi, "SError");
990 } else
991 ata_ehi_push_desc(ehi, "unknown");
992 ata_port_freeze(ap);
993 continue;
994 }
995
996 if (status & (NV_ADMA_STAT_DONE |
997 NV_ADMA_STAT_CPBERR |
998 NV_ADMA_STAT_CMD_COMPLETE)) {
999 u32 check_commands = notifier_clears[i];
Tejun Heo1aadf5c2010-06-25 15:03:34 +02001000 u32 done_mask = 0;
Tejun Heo752e3862010-06-25 15:02:59 +02001001 int pos, rc;
Tejun Heo3e4ec342010-05-10 21:41:30 +02001002
1003 if (status & NV_ADMA_STAT_CPBERR) {
1004 /* check all active commands */
Jeff Garzik2dcb4072007-10-19 06:42:56 -04001005 if (ata_tag_valid(ap->link.active_tag))
Tejun Heo3e4ec342010-05-10 21:41:30 +02001006 check_commands = 1 <<
1007 ap->link.active_tag;
1008 else
1009 check_commands = ap->link.sactive;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001010 }
1011
Tejun Heo3e4ec342010-05-10 21:41:30 +02001012 /* check CPBs for completed commands */
Tejun Heo752e3862010-06-25 15:02:59 +02001013 while ((pos = ffs(check_commands))) {
Tejun Heo3e4ec342010-05-10 21:41:30 +02001014 pos--;
Tejun Heo752e3862010-06-25 15:02:59 +02001015 rc = nv_adma_check_cpb(ap, pos,
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001016 notifier_error & (1 << pos));
Tejun Heo1aadf5c2010-06-25 15:03:34 +02001017 if (rc > 0)
1018 done_mask |= 1 << pos;
1019 else if (unlikely(rc < 0))
Tejun Heo752e3862010-06-25 15:02:59 +02001020 check_commands = 0;
Tejun Heo3e4ec342010-05-10 21:41:30 +02001021 check_commands &= ~(1 << pos);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001022 }
Tejun Heo1aadf5c2010-06-25 15:03:34 +02001023 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001024 }
1025 }
Jeff Garzikf20b16f2006-12-11 11:14:06 -05001026
Jeff Garzikb4479162007-10-25 20:47:30 -04001027 if (notifier_clears[0] || notifier_clears[1]) {
Robert Hancock2dec7552006-11-26 14:20:19 -06001028 /* Note: Both notifier clear registers must be written
1029 if either is set, even if one is zero, according to NVIDIA. */
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001030 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1031 writel(notifier_clears[0], pp->notifier_clear_block);
1032 pp = host->ports[1]->private_data;
1033 writel(notifier_clears[1], pp->notifier_clear_block);
Robert Hancock2dec7552006-11-26 14:20:19 -06001034 }
Robert Hancockfbbb2622006-10-27 19:08:41 -07001035
1036 spin_unlock(&host->lock);
1037
1038 return IRQ_RETVAL(handled);
1039}
1040
Robert Hancock53014e22007-05-05 15:36:36 -06001041static void nv_adma_freeze(struct ata_port *ap)
1042{
1043 struct nv_adma_port_priv *pp = ap->private_data;
1044 void __iomem *mmio = pp->ctl_block;
1045 u16 tmp;
1046
1047 nv_ck804_freeze(ap);
1048
1049 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1050 return;
1051
1052 /* clear any outstanding CK804 notifications */
Jeff Garzik2dcb4072007-10-19 06:42:56 -04001053 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
Robert Hancock53014e22007-05-05 15:36:36 -06001054 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1055
1056 /* Disable interrupt */
1057 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik2dcb4072007-10-19 06:42:56 -04001058 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
Robert Hancock53014e22007-05-05 15:36:36 -06001059 mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001060 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancock53014e22007-05-05 15:36:36 -06001061}
1062
1063static void nv_adma_thaw(struct ata_port *ap)
1064{
1065 struct nv_adma_port_priv *pp = ap->private_data;
1066 void __iomem *mmio = pp->ctl_block;
1067 u16 tmp;
1068
1069 nv_ck804_thaw(ap);
1070
1071 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1072 return;
1073
1074 /* Enable interrupt */
1075 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik2dcb4072007-10-19 06:42:56 -04001076 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
Robert Hancock53014e22007-05-05 15:36:36 -06001077 mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001078 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancock53014e22007-05-05 15:36:36 -06001079}
1080
Robert Hancockfbbb2622006-10-27 19:08:41 -07001081static void nv_adma_irq_clear(struct ata_port *ap)
1082{
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001083 struct nv_adma_port_priv *pp = ap->private_data;
1084 void __iomem *mmio = pp->ctl_block;
Robert Hancock53014e22007-05-05 15:36:36 -06001085 u32 notifier_clears[2];
1086
1087 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
Tejun Heo37f65b82010-05-19 22:10:20 +02001088 ata_bmdma_irq_clear(ap);
Robert Hancock53014e22007-05-05 15:36:36 -06001089 return;
1090 }
1091
1092 /* clear any outstanding CK804 notifications */
Jeff Garzik2dcb4072007-10-19 06:42:56 -04001093 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
Robert Hancock53014e22007-05-05 15:36:36 -06001094 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001095
1096 /* clear ADMA status */
Robert Hancock53014e22007-05-05 15:36:36 -06001097 writew(0xffff, mmio + NV_ADMA_STAT);
Jeff Garzika617c092007-05-21 20:14:23 -04001098
Robert Hancock53014e22007-05-05 15:36:36 -06001099 /* clear notifiers - note both ports need to be written with
1100 something even though we are only clearing on one */
1101 if (ap->port_no == 0) {
1102 notifier_clears[0] = 0xFFFFFFFF;
1103 notifier_clears[1] = 0;
1104 } else {
1105 notifier_clears[0] = 0;
1106 notifier_clears[1] = 0xFFFFFFFF;
1107 }
1108 pp = ap->host->ports[0]->private_data;
1109 writel(notifier_clears[0], pp->notifier_clear_block);
1110 pp = ap->host->ports[1]->private_data;
1111 writel(notifier_clears[1], pp->notifier_clear_block);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001112}
1113
Robert Hancockf5ecac22007-02-20 21:49:10 -06001114static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001115{
Robert Hancockf5ecac22007-02-20 21:49:10 -06001116 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001117
Jeff Garzikb4479162007-10-25 20:47:30 -04001118 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
Tejun Heofe06e5f2010-05-10 21:41:39 +02001119 ata_bmdma_post_internal_cmd(qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001120}
1121
1122static int nv_adma_port_start(struct ata_port *ap)
1123{
1124 struct device *dev = ap->host->dev;
1125 struct nv_adma_port_priv *pp;
1126 int rc;
1127 void *mem;
1128 dma_addr_t mem_dma;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001129 void __iomem *mmio;
Robert Hancock8959d302008-02-04 19:39:02 -06001130 struct pci_dev *pdev = to_pci_dev(dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001131 u16 tmp;
1132
1133 VPRINTK("ENTER\n");
1134
Robert Hancock8959d302008-02-04 19:39:02 -06001135 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1136 pad buffers */
1137 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1138 if (rc)
1139 return rc;
1140 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1141 if (rc)
1142 return rc;
1143
Tejun Heoc7087652010-05-10 21:41:34 +02001144 /* we might fallback to bmdma, allocate bmdma resources */
1145 rc = ata_bmdma_port_start(ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001146 if (rc)
1147 return rc;
1148
Tejun Heo24dc5f32007-01-20 16:00:28 +09001149 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1150 if (!pp)
1151 return -ENOMEM;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001152
Tejun Heo0d5ff562007-02-01 15:06:36 +09001153 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001154 ap->port_no * NV_ADMA_PORT_SIZE;
1155 pp->ctl_block = mmio;
Tejun Heo0d5ff562007-02-01 15:06:36 +09001156 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001157 pp->notifier_clear_block = pp->gen_block +
1158 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1159
Robert Hancock8959d302008-02-04 19:39:02 -06001160 /* Now that the legacy PRD and padding buffer are allocated we can
1161 safely raise the DMA mask to allocate the CPB/APRD table.
1162 These are allowed to fail since we store the value that ends up
1163 being used to set as the bounce limit in slave_config later if
1164 needed. */
1165 pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1166 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1167 pp->adma_dma_mask = *dev->dma_mask;
1168
Tejun Heo24dc5f32007-01-20 16:00:28 +09001169 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1170 &mem_dma, GFP_KERNEL);
1171 if (!mem)
1172 return -ENOMEM;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001173 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1174
1175 /*
1176 * First item in chunk of DMA memory:
1177 * 128-byte command parameter block (CPB)
1178 * one for each command tag
1179 */
1180 pp->cpb = mem;
1181 pp->cpb_dma = mem_dma;
1182
1183 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001184 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001185
1186 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1187 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1188
1189 /*
1190 * Second item: block of ADMA_SGTBL_LEN s/g entries
1191 */
1192 pp->aprd = mem;
1193 pp->aprd_dma = mem_dma;
1194
1195 ap->private_data = pp;
1196
1197 /* clear any outstanding interrupt conditions */
1198 writew(0xffff, mmio + NV_ADMA_STAT);
1199
1200 /* initialize port variables */
1201 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1202
1203 /* clear CPB fetch count */
1204 writew(0, mmio + NV_ADMA_CPB_COUNT);
1205
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001206 /* clear GO for register mode, enable interrupt */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001207 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001208 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1209 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001210
1211 tmp = readw(mmio + NV_ADMA_CTL);
1212 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001213 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001214 udelay(1);
1215 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001216 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001217
1218 return 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001219}
1220
1221static void nv_adma_port_stop(struct ata_port *ap)
1222{
Robert Hancockfbbb2622006-10-27 19:08:41 -07001223 struct nv_adma_port_priv *pp = ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001224 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001225
1226 VPRINTK("ENTER\n");
Robert Hancockfbbb2622006-10-27 19:08:41 -07001227 writew(0, mmio + NV_ADMA_CTL);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001228}
1229
Tejun Heo438ac6d2007-03-02 17:31:26 +09001230#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001231static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1232{
1233 struct nv_adma_port_priv *pp = ap->private_data;
1234 void __iomem *mmio = pp->ctl_block;
1235
1236 /* Go to register mode - clears GO */
1237 nv_adma_register_mode(ap);
1238
1239 /* clear CPB fetch count */
1240 writew(0, mmio + NV_ADMA_CPB_COUNT);
1241
1242 /* disable interrupt, shut down port */
1243 writew(0, mmio + NV_ADMA_CTL);
1244
1245 return 0;
1246}
1247
1248static int nv_adma_port_resume(struct ata_port *ap)
1249{
1250 struct nv_adma_port_priv *pp = ap->private_data;
1251 void __iomem *mmio = pp->ctl_block;
1252 u16 tmp;
1253
1254 /* set CPB block location */
1255 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001256 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001257
1258 /* clear any outstanding interrupt conditions */
1259 writew(0xffff, mmio + NV_ADMA_STAT);
1260
1261 /* initialize port variables */
1262 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1263
1264 /* clear CPB fetch count */
1265 writew(0, mmio + NV_ADMA_CPB_COUNT);
1266
1267 /* clear GO for register mode, enable interrupt */
1268 tmp = readw(mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001269 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1270 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001271
1272 tmp = readw(mmio + NV_ADMA_CTL);
1273 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001274 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001275 udelay(1);
1276 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001277 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001278
1279 return 0;
1280}
Tejun Heo438ac6d2007-03-02 17:31:26 +09001281#endif
Robert Hancockfbbb2622006-10-27 19:08:41 -07001282
Tejun Heo9a829cc2007-04-17 23:44:08 +09001283static void nv_adma_setup_port(struct ata_port *ap)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001284{
Tejun Heo9a829cc2007-04-17 23:44:08 +09001285 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1286 struct ata_ioports *ioport = &ap->ioaddr;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001287
1288 VPRINTK("ENTER\n");
1289
Tejun Heo9a829cc2007-04-17 23:44:08 +09001290 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001291
Tejun Heo0d5ff562007-02-01 15:06:36 +09001292 ioport->cmd_addr = mmio;
1293 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001294 ioport->error_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001295 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1296 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1297 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1298 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1299 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1300 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001301 ioport->status_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001302 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001303 ioport->altstatus_addr =
Tejun Heo0d5ff562007-02-01 15:06:36 +09001304 ioport->ctl_addr = mmio + 0x20;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001305}
1306
Tejun Heo9a829cc2007-04-17 23:44:08 +09001307static int nv_adma_host_init(struct ata_host *host)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001308{
Tejun Heo9a829cc2007-04-17 23:44:08 +09001309 struct pci_dev *pdev = to_pci_dev(host->dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001310 unsigned int i;
1311 u32 tmp32;
1312
1313 VPRINTK("ENTER\n");
1314
1315 /* enable ADMA on the ports */
1316 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1317 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1318 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1319 NV_MCP_SATA_CFG_20_PORT1_EN |
1320 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1321
1322 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1323
Tejun Heo9a829cc2007-04-17 23:44:08 +09001324 for (i = 0; i < host->n_ports; i++)
1325 nv_adma_setup_port(host->ports[i]);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001326
Robert Hancockfbbb2622006-10-27 19:08:41 -07001327 return 0;
1328}
1329
1330static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1331 struct scatterlist *sg,
1332 int idx,
1333 struct nv_adma_prd *aprd)
1334{
Robert Hancock41949ed2007-02-19 19:02:27 -06001335 u8 flags = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001336 if (qc->tf.flags & ATA_TFLAG_WRITE)
1337 flags |= NV_APRD_WRITE;
1338 if (idx == qc->n_elem - 1)
1339 flags |= NV_APRD_END;
1340 else if (idx != 4)
1341 flags |= NV_APRD_CONT;
1342
1343 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1344 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
Robert Hancock2dec7552006-11-26 14:20:19 -06001345 aprd->flags = flags;
Robert Hancock41949ed2007-02-19 19:02:27 -06001346 aprd->packet_len = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001347}
1348
1349static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1350{
1351 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001352 struct nv_adma_prd *aprd;
1353 struct scatterlist *sg;
Tejun Heoff2aeb12007-12-05 16:43:11 +09001354 unsigned int si;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001355
1356 VPRINTK("ENTER\n");
1357
Tejun Heoff2aeb12007-12-05 16:43:11 +09001358 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1359 aprd = (si < 5) ? &cpb->aprd[si] :
1360 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1361 nv_adma_fill_aprd(qc, sg, si, aprd);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001362 }
Tejun Heoff2aeb12007-12-05 16:43:11 +09001363 if (si > 5)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001364 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
Robert Hancock41949ed2007-02-19 19:02:27 -06001365 else
1366 cpb->next_aprd = cpu_to_le64(0);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001367}
1368
Robert Hancock382a6652007-02-05 16:26:02 -08001369static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1370{
1371 struct nv_adma_port_priv *pp = qc->ap->private_data;
1372
1373 /* ADMA engine can only be used for non-ATAPI DMA commands,
Robert Hancock3f3debd2007-11-25 16:59:36 -06001374 or interrupt-driven no-data commands. */
Jeff Garzikb4479162007-10-25 20:47:30 -04001375 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
Robert Hancock3f3debd2007-11-25 16:59:36 -06001376 (qc->tf.flags & ATA_TFLAG_POLLING))
Robert Hancock382a6652007-02-05 16:26:02 -08001377 return 1;
1378
Jeff Garzikb4479162007-10-25 20:47:30 -04001379 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
Robert Hancock382a6652007-02-05 16:26:02 -08001380 (qc->tf.protocol == ATA_PROT_NODATA))
1381 return 0;
1382
1383 return 1;
1384}
1385
Robert Hancockfbbb2622006-10-27 19:08:41 -07001386static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1387{
1388 struct nv_adma_port_priv *pp = qc->ap->private_data;
1389 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1390 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
Robert Hancockfbbb2622006-10-27 19:08:41 -07001391 NV_CPB_CTL_IEN;
1392
Robert Hancock382a6652007-02-05 16:26:02 -08001393 if (nv_adma_use_reg_mode(qc)) {
Robert Hancock3f3debd2007-11-25 16:59:36 -06001394 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1395 (qc->flags & ATA_QCFLAG_DMAMAP));
Robert Hancock2dec7552006-11-26 14:20:19 -06001396 nv_adma_register_mode(qc->ap);
Tejun Heof47451c2010-05-10 21:41:40 +02001397 ata_bmdma_qc_prep(qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001398 return;
1399 }
1400
Robert Hancock41949ed2007-02-19 19:02:27 -06001401 cpb->resp_flags = NV_CPB_RESP_DONE;
1402 wmb();
1403 cpb->ctl_flags = 0;
1404 wmb();
Robert Hancockfbbb2622006-10-27 19:08:41 -07001405
1406 cpb->len = 3;
1407 cpb->tag = qc->tag;
1408 cpb->next_cpb_idx = 0;
1409
1410 /* turn on NCQ flags for NCQ commands */
1411 if (qc->tf.protocol == ATA_PROT_NCQ)
1412 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1413
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001414 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1415
Robert Hancockfbbb2622006-10-27 19:08:41 -07001416 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1417
Jeff Garzikb4479162007-10-25 20:47:30 -04001418 if (qc->flags & ATA_QCFLAG_DMAMAP) {
Robert Hancock382a6652007-02-05 16:26:02 -08001419 nv_adma_fill_sg(qc, cpb);
1420 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1421 } else
1422 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001423
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001424 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1425 until we are finished filling in all of the contents */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001426 wmb();
1427 cpb->ctl_flags = ctl_flags;
Robert Hancock41949ed2007-02-19 19:02:27 -06001428 wmb();
1429 cpb->resp_flags = 0;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001430}
1431
1432static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1433{
Robert Hancock2dec7552006-11-26 14:20:19 -06001434 struct nv_adma_port_priv *pp = qc->ap->private_data;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001435 void __iomem *mmio = pp->ctl_block;
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001436 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001437
1438 VPRINTK("ENTER\n");
1439
Robert Hancock3f3debd2007-11-25 16:59:36 -06001440 /* We can't handle result taskfile with NCQ commands, since
1441 retrieving the taskfile switches us out of ADMA mode and would abort
1442 existing commands. */
1443 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1444 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
Joe Perchesa9a79df2011-04-15 15:51:59 -07001445 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");
Robert Hancock3f3debd2007-11-25 16:59:36 -06001446 return AC_ERR_SYSTEM;
1447 }
1448
Robert Hancock382a6652007-02-05 16:26:02 -08001449 if (nv_adma_use_reg_mode(qc)) {
Robert Hancockfbbb2622006-10-27 19:08:41 -07001450 /* use ATA register mode */
Robert Hancock382a6652007-02-05 16:26:02 -08001451 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
Robert Hancock3f3debd2007-11-25 16:59:36 -06001452 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1453 (qc->flags & ATA_QCFLAG_DMAMAP));
Robert Hancockfbbb2622006-10-27 19:08:41 -07001454 nv_adma_register_mode(qc->ap);
Tejun Heo360ff782010-05-10 21:41:42 +02001455 return ata_bmdma_qc_issue(qc);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001456 } else
1457 nv_adma_mode(qc->ap);
1458
1459 /* write append register, command tag in lower 8 bits
1460 and (number of cpbs to append -1) in top 8 bits */
1461 wmb();
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001462
Jeff Garzikb4479162007-10-25 20:47:30 -04001463 if (curr_ncq != pp->last_issue_ncq) {
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001464 /* Seems to need some delay before switching between NCQ and
1465 non-NCQ commands, else we get command timeouts and such. */
Robert Hancock5e5c74a2007-02-19 18:42:30 -06001466 udelay(20);
1467 pp->last_issue_ncq = curr_ncq;
1468 }
1469
Robert Hancockfbbb2622006-10-27 19:08:41 -07001470 writew(qc->tag, mmio + NV_ADMA_APPEND);
1471
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001472 DPRINTK("Issued tag %u\n", qc->tag);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001473
1474 return 0;
1475}
1476
David Howells7d12e782006-10-05 14:55:46 +01001477static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478{
Jeff Garzikcca39742006-08-24 03:19:22 -04001479 struct ata_host *host = dev_instance;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480 unsigned int i;
1481 unsigned int handled = 0;
1482 unsigned long flags;
1483
Jeff Garzikcca39742006-08-24 03:19:22 -04001484 spin_lock_irqsave(&host->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485
Jeff Garzikcca39742006-08-24 03:19:22 -04001486 for (i = 0; i < host->n_ports; i++) {
Tejun Heo3e4ec342010-05-10 21:41:30 +02001487 struct ata_port *ap = host->ports[i];
1488 struct ata_queued_cmd *qc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489
Tejun Heo3e4ec342010-05-10 21:41:30 +02001490 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1491 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
Tejun Heoc3b28892010-05-19 22:10:21 +02001492 handled += ata_bmdma_port_intr(ap, qc);
Tejun Heo3e4ec342010-05-10 21:41:30 +02001493 } else {
1494 /*
1495 * No request pending? Clear interrupt status
1496 * anyway, in case there's one pending.
1497 */
1498 ap->ops->sff_check_status(ap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 }
1501
Jeff Garzikcca39742006-08-24 03:19:22 -04001502 spin_unlock_irqrestore(&host->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503
1504 return IRQ_RETVAL(handled);
1505}
1506
Jeff Garzikcca39742006-08-24 03:19:22 -04001507static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
Tejun Heoada364e2006-06-17 15:49:56 +09001508{
1509 int i, handled = 0;
1510
Jeff Garzikcca39742006-08-24 03:19:22 -04001511 for (i = 0; i < host->n_ports; i++) {
Tejun Heo3e4ec342010-05-10 21:41:30 +02001512 handled += nv_host_intr(host->ports[i], irq_stat);
Tejun Heoada364e2006-06-17 15:49:56 +09001513 irq_stat >>= NV_INT_PORT_SHIFT;
1514 }
1515
1516 return IRQ_RETVAL(handled);
1517}
1518
David Howells7d12e782006-10-05 14:55:46 +01001519static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
Tejun Heoada364e2006-06-17 15:49:56 +09001520{
Jeff Garzikcca39742006-08-24 03:19:22 -04001521 struct ata_host *host = dev_instance;
Tejun Heoada364e2006-06-17 15:49:56 +09001522 u8 irq_stat;
1523 irqreturn_t ret;
1524
Jeff Garzikcca39742006-08-24 03:19:22 -04001525 spin_lock(&host->lock);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001526 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
Jeff Garzikcca39742006-08-24 03:19:22 -04001527 ret = nv_do_interrupt(host, irq_stat);
1528 spin_unlock(&host->lock);
Tejun Heoada364e2006-06-17 15:49:56 +09001529
1530 return ret;
1531}
1532
David Howells7d12e782006-10-05 14:55:46 +01001533static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
Tejun Heoada364e2006-06-17 15:49:56 +09001534{
Jeff Garzikcca39742006-08-24 03:19:22 -04001535 struct ata_host *host = dev_instance;
Tejun Heoada364e2006-06-17 15:49:56 +09001536 u8 irq_stat;
1537 irqreturn_t ret;
1538
Jeff Garzikcca39742006-08-24 03:19:22 -04001539 spin_lock(&host->lock);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001540 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
Jeff Garzikcca39742006-08-24 03:19:22 -04001541 ret = nv_do_interrupt(host, irq_stat);
1542 spin_unlock(&host->lock);
Tejun Heoada364e2006-06-17 15:49:56 +09001543
1544 return ret;
1545}
1546
Tejun Heo82ef04f2008-07-31 17:02:40 +09001547static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549 if (sc_reg > SCR_CONTROL)
Tejun Heoda3dbb12007-07-16 14:29:40 +09001550 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551
Tejun Heo82ef04f2008-07-31 17:02:40 +09001552 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
Tejun Heoda3dbb12007-07-16 14:29:40 +09001553 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554}
1555
Tejun Heo82ef04f2008-07-31 17:02:40 +09001556static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558 if (sc_reg > SCR_CONTROL)
Tejun Heoda3dbb12007-07-16 14:29:40 +09001559 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560
Tejun Heo82ef04f2008-07-31 17:02:40 +09001561 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
Tejun Heoda3dbb12007-07-16 14:29:40 +09001562 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563}
1564
Tejun Heo7f4774b2009-06-10 16:29:07 +09001565static int nv_hardreset(struct ata_link *link, unsigned int *class,
1566 unsigned long deadline)
Tejun Heoe8caa3c2009-01-25 11:25:22 +09001567{
Tejun Heo7f4774b2009-06-10 16:29:07 +09001568 struct ata_eh_context *ehc = &link->eh_context;
Tejun Heoe8caa3c2009-01-25 11:25:22 +09001569
Tejun Heo7f4774b2009-06-10 16:29:07 +09001570 /* Do hardreset iff it's post-boot probing, please read the
1571 * comment above port ops for details.
1572 */
1573 if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1574 !ata_dev_enabled(link->device))
1575 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1576 NULL, NULL);
Tejun Heo6489e322009-10-14 11:18:28 +09001577 else {
1578 const unsigned long *timing = sata_ehc_deb_timing(ehc);
1579 int rc;
1580
1581 if (!(ehc->i.flags & ATA_EHI_QUIET))
Joe Perchesa9a79df2011-04-15 15:51:59 -07001582 ata_link_info(link,
1583 "nv: skipping hardreset on occupied port\n");
Tejun Heo6489e322009-10-14 11:18:28 +09001584
1585 /* make sure the link is online */
1586 rc = sata_link_resume(link, timing, deadline);
1587 /* whine about phy resume failure but proceed */
1588 if (rc && rc != -EOPNOTSUPP)
Joe Perchesa9a79df2011-04-15 15:51:59 -07001589 ata_link_warn(link, "failed to resume link (errno=%d)\n",
1590 rc);
Tejun Heo6489e322009-10-14 11:18:28 +09001591 }
Tejun Heo7f4774b2009-06-10 16:29:07 +09001592
1593 /* device signature acquisition is unreliable */
1594 return -EAGAIN;
Tejun Heoe8caa3c2009-01-25 11:25:22 +09001595}
1596
Tejun Heo39f87582006-06-17 15:49:56 +09001597static void nv_nf2_freeze(struct ata_port *ap)
1598{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001599 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
Tejun Heo39f87582006-06-17 15:49:56 +09001600 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1601 u8 mask;
1602
Tejun Heo0d5ff562007-02-01 15:06:36 +09001603 mask = ioread8(scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001604 mask &= ~(NV_INT_ALL << shift);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001605 iowrite8(mask, scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001606}
1607
1608static void nv_nf2_thaw(struct ata_port *ap)
1609{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001610 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
Tejun Heo39f87582006-06-17 15:49:56 +09001611 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1612 u8 mask;
1613
Tejun Heo0d5ff562007-02-01 15:06:36 +09001614 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
Tejun Heo39f87582006-06-17 15:49:56 +09001615
Tejun Heo0d5ff562007-02-01 15:06:36 +09001616 mask = ioread8(scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001617 mask |= (NV_INT_MASK << shift);
Tejun Heo0d5ff562007-02-01 15:06:36 +09001618 iowrite8(mask, scr_addr + NV_INT_ENABLE);
Tejun Heo39f87582006-06-17 15:49:56 +09001619}
1620
1621static void nv_ck804_freeze(struct ata_port *ap)
1622{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001623 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
Tejun Heo39f87582006-06-17 15:49:56 +09001624 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1625 u8 mask;
1626
1627 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1628 mask &= ~(NV_INT_ALL << shift);
1629 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1630}
1631
1632static void nv_ck804_thaw(struct ata_port *ap)
1633{
Tejun Heo0d5ff562007-02-01 15:06:36 +09001634 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
Tejun Heo39f87582006-06-17 15:49:56 +09001635 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1636 u8 mask;
1637
1638 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1639
1640 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1641 mask |= (NV_INT_MASK << shift);
1642 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1643}
1644
Kuan Luof140f0f2007-10-15 15:16:53 -04001645static void nv_mcp55_freeze(struct ata_port *ap)
1646{
1647 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1648 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1649 u32 mask;
1650
1651 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1652
1653 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1654 mask &= ~(NV_INT_ALL_MCP55 << shift);
1655 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
Kuan Luof140f0f2007-10-15 15:16:53 -04001656}
1657
1658static void nv_mcp55_thaw(struct ata_port *ap)
1659{
1660 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1661 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1662 u32 mask;
1663
1664 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1665
1666 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1667 mask |= (NV_INT_MASK_MCP55 << shift);
1668 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
Kuan Luof140f0f2007-10-15 15:16:53 -04001669}
1670
Robert Hancockfbbb2622006-10-27 19:08:41 -07001671static void nv_adma_error_handler(struct ata_port *ap)
1672{
1673 struct nv_adma_port_priv *pp = ap->private_data;
Jeff Garzikb4479162007-10-25 20:47:30 -04001674 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
Robert Hancockcdf56bc2007-01-03 18:13:57 -06001675 void __iomem *mmio = pp->ctl_block;
Robert Hancockfbbb2622006-10-27 19:08:41 -07001676 int i;
1677 u16 tmp;
Jeff Garzika84471f2007-02-26 05:51:33 -05001678
Jeff Garzikb4479162007-10-25 20:47:30 -04001679 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
Robert Hancock2cb27852007-02-11 18:34:44 -06001680 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1681 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1682 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1683 u32 status = readw(mmio + NV_ADMA_STAT);
Robert Hancock08af7412007-02-19 19:01:59 -06001684 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1685 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
Robert Hancock2cb27852007-02-11 18:34:44 -06001686
Joe Perchesa9a79df2011-04-15 15:51:59 -07001687 ata_port_err(ap,
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001688 "EH in ADMA mode, notifier 0x%X "
Robert Hancock08af7412007-02-19 19:01:59 -06001689 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1690 "next cpb count 0x%X next cpb idx 0x%x\n",
1691 notifier, notifier_error, gen_ctl, status,
1692 cpb_count, next_cpb_idx);
Robert Hancock2cb27852007-02-11 18:34:44 -06001693
Jeff Garzikb4479162007-10-25 20:47:30 -04001694 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
Robert Hancock2cb27852007-02-11 18:34:44 -06001695 struct nv_adma_cpb *cpb = &pp->cpb[i];
Jeff Garzikb4479162007-10-25 20:47:30 -04001696 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001697 ap->link.sactive & (1 << i))
Joe Perchesa9a79df2011-04-15 15:51:59 -07001698 ata_port_err(ap,
Robert Hancock2cb27852007-02-11 18:34:44 -06001699 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1700 i, cpb->ctl_flags, cpb->resp_flags);
1701 }
1702 }
Robert Hancockfbbb2622006-10-27 19:08:41 -07001703
Robert Hancockfbbb2622006-10-27 19:08:41 -07001704 /* Push us back into port register mode for error handling. */
1705 nv_adma_register_mode(ap);
1706
Jeff Garzik5796d1c2007-10-26 00:03:37 -04001707 /* Mark all of the CPBs as invalid to prevent them from
1708 being executed */
Jeff Garzikb4479162007-10-25 20:47:30 -04001709 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
Robert Hancockfbbb2622006-10-27 19:08:41 -07001710 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1711
1712 /* clear CPB fetch count */
1713 writew(0, mmio + NV_ADMA_CPB_COUNT);
1714
1715 /* Reset channel */
1716 tmp = readw(mmio + NV_ADMA_CTL);
1717 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzikb4479162007-10-25 20:47:30 -04001718 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001719 udelay(1);
1720 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
Jeff Garzikb4479162007-10-25 20:47:30 -04001721 readw(mmio + NV_ADMA_CTL); /* flush posted write */
Robert Hancockfbbb2622006-10-27 19:08:41 -07001722 }
1723
Tejun Heofe06e5f2010-05-10 21:41:39 +02001724 ata_bmdma_error_handler(ap);
Robert Hancockfbbb2622006-10-27 19:08:41 -07001725}
1726
Kuan Luof140f0f2007-10-15 15:16:53 -04001727static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1728{
1729 struct nv_swncq_port_priv *pp = ap->private_data;
1730 struct defer_queue *dq = &pp->defer_queue;
1731
1732 /* queue is full */
1733 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1734 dq->defer_bits |= (1 << qc->tag);
1735 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1736}
1737
1738static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1739{
1740 struct nv_swncq_port_priv *pp = ap->private_data;
1741 struct defer_queue *dq = &pp->defer_queue;
1742 unsigned int tag;
1743
1744 if (dq->head == dq->tail) /* null queue */
1745 return NULL;
1746
1747 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1748 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1749 WARN_ON(!(dq->defer_bits & (1 << tag)));
1750 dq->defer_bits &= ~(1 << tag);
1751
1752 return ata_qc_from_tag(ap, tag);
1753}
1754
1755static void nv_swncq_fis_reinit(struct ata_port *ap)
1756{
1757 struct nv_swncq_port_priv *pp = ap->private_data;
1758
1759 pp->dhfis_bits = 0;
1760 pp->dmafis_bits = 0;
1761 pp->sdbfis_bits = 0;
1762 pp->ncq_flags = 0;
1763}
1764
1765static void nv_swncq_pp_reinit(struct ata_port *ap)
1766{
1767 struct nv_swncq_port_priv *pp = ap->private_data;
1768 struct defer_queue *dq = &pp->defer_queue;
1769
1770 dq->head = 0;
1771 dq->tail = 0;
1772 dq->defer_bits = 0;
1773 pp->qc_active = 0;
1774 pp->last_issue_tag = ATA_TAG_POISON;
1775 nv_swncq_fis_reinit(ap);
1776}
1777
1778static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1779{
1780 struct nv_swncq_port_priv *pp = ap->private_data;
1781
1782 writew(fis, pp->irq_block);
1783}
1784
1785static void __ata_bmdma_stop(struct ata_port *ap)
1786{
1787 struct ata_queued_cmd qc;
1788
1789 qc.ap = ap;
1790 ata_bmdma_stop(&qc);
1791}
1792
1793static void nv_swncq_ncq_stop(struct ata_port *ap)
1794{
1795 struct nv_swncq_port_priv *pp = ap->private_data;
1796 unsigned int i;
1797 u32 sactive;
1798 u32 done_mask;
1799
Joe Perchesa9a79df2011-04-15 15:51:59 -07001800 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1801 ap->qc_active, ap->link.sactive);
1802 ata_port_err(ap,
Kuan Luof140f0f2007-10-15 15:16:53 -04001803 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1804 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1805 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1806 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1807
Joe Perchesa9a79df2011-04-15 15:51:59 -07001808 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n",
1809 ap->ops->sff_check_status(ap),
1810 ioread8(ap->ioaddr.error_addr));
Kuan Luof140f0f2007-10-15 15:16:53 -04001811
1812 sactive = readl(pp->sactive_block);
1813 done_mask = pp->qc_active ^ sactive;
1814
Joe Perchesa9a79df2011-04-15 15:51:59 -07001815 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n");
Kuan Luof140f0f2007-10-15 15:16:53 -04001816 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1817 u8 err = 0;
1818 if (pp->qc_active & (1 << i))
1819 err = 0;
1820 else if (done_mask & (1 << i))
1821 err = 1;
1822 else
1823 continue;
1824
Joe Perchesa9a79df2011-04-15 15:51:59 -07001825 ata_port_err(ap,
1826 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1827 (pp->dhfis_bits >> i) & 0x1,
1828 (pp->dmafis_bits >> i) & 0x1,
1829 (pp->sdbfis_bits >> i) & 0x1,
1830 (sactive >> i) & 0x1,
1831 (err ? "error! tag doesn't exit" : " "));
Kuan Luof140f0f2007-10-15 15:16:53 -04001832 }
1833
1834 nv_swncq_pp_reinit(ap);
Tejun Heo5682ed32008-04-07 22:47:16 +09001835 ap->ops->sff_irq_clear(ap);
Kuan Luof140f0f2007-10-15 15:16:53 -04001836 __ata_bmdma_stop(ap);
1837 nv_swncq_irq_clear(ap, 0xffff);
1838}
1839
1840static void nv_swncq_error_handler(struct ata_port *ap)
1841{
1842 struct ata_eh_context *ehc = &ap->link.eh_context;
1843
1844 if (ap->link.sactive) {
1845 nv_swncq_ncq_stop(ap);
Tejun Heocf480622008-01-24 00:05:14 +09001846 ehc->i.action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04001847 }
1848
Tejun Heofe06e5f2010-05-10 21:41:39 +02001849 ata_bmdma_error_handler(ap);
Kuan Luof140f0f2007-10-15 15:16:53 -04001850}
1851
1852#ifdef CONFIG_PM
1853static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1854{
1855 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1856 u32 tmp;
1857
1858 /* clear irq */
1859 writel(~0, mmio + NV_INT_STATUS_MCP55);
1860
1861 /* disable irq */
1862 writel(0, mmio + NV_INT_ENABLE_MCP55);
1863
1864 /* disable swncq */
1865 tmp = readl(mmio + NV_CTL_MCP55);
1866 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1867 writel(tmp, mmio + NV_CTL_MCP55);
1868
1869 return 0;
1870}
1871
1872static int nv_swncq_port_resume(struct ata_port *ap)
1873{
1874 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1875 u32 tmp;
1876
1877 /* clear irq */
1878 writel(~0, mmio + NV_INT_STATUS_MCP55);
1879
1880 /* enable irq */
1881 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1882
1883 /* enable swncq */
1884 tmp = readl(mmio + NV_CTL_MCP55);
1885 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1886
1887 return 0;
1888}
1889#endif
1890
1891static void nv_swncq_host_init(struct ata_host *host)
1892{
1893 u32 tmp;
1894 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1895 struct pci_dev *pdev = to_pci_dev(host->dev);
1896 u8 regval;
1897
1898 /* disable ECO 398 */
1899 pci_read_config_byte(pdev, 0x7f, &regval);
1900 regval &= ~(1 << 7);
1901 pci_write_config_byte(pdev, 0x7f, regval);
1902
1903 /* enable swncq */
1904 tmp = readl(mmio + NV_CTL_MCP55);
1905 VPRINTK("HOST_CTL:0x%X\n", tmp);
1906 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1907
1908 /* enable irq intr */
1909 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1910 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1911 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1912
1913 /* clear port irq */
1914 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1915}
1916
1917static int nv_swncq_slave_config(struct scsi_device *sdev)
1918{
1919 struct ata_port *ap = ata_shost_to_port(sdev->host);
1920 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1921 struct ata_device *dev;
1922 int rc;
1923 u8 rev;
1924 u8 check_maxtor = 0;
1925 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1926
1927 rc = ata_scsi_slave_config(sdev);
1928 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1929 /* Not a proper libata device, ignore */
1930 return rc;
1931
1932 dev = &ap->link.device[sdev->id];
1933 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1934 return rc;
1935
1936 /* if MCP51 and Maxtor, then disable ncq */
1937 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1938 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1939 check_maxtor = 1;
1940
1941 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1942 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1943 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1944 pci_read_config_byte(pdev, 0x8, &rev);
1945 if (rev <= 0xa2)
1946 check_maxtor = 1;
1947 }
1948
1949 if (!check_maxtor)
1950 return rc;
1951
1952 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1953
1954 if (strncmp(model_num, "Maxtor", 6) == 0) {
Mike Christiee881a172009-10-15 17:46:39 -07001955 ata_scsi_change_queue_depth(sdev, 1, SCSI_QDEPTH_DEFAULT);
Joe Perchesa9a79df2011-04-15 15:51:59 -07001956 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n",
1957 sdev->queue_depth);
Kuan Luof140f0f2007-10-15 15:16:53 -04001958 }
1959
1960 return rc;
1961}
1962
1963static int nv_swncq_port_start(struct ata_port *ap)
1964{
1965 struct device *dev = ap->host->dev;
1966 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1967 struct nv_swncq_port_priv *pp;
1968 int rc;
1969
Tejun Heoc7087652010-05-10 21:41:34 +02001970 /* we might fallback to bmdma, allocate bmdma resources */
1971 rc = ata_bmdma_port_start(ap);
Kuan Luof140f0f2007-10-15 15:16:53 -04001972 if (rc)
1973 return rc;
1974
1975 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1976 if (!pp)
1977 return -ENOMEM;
1978
1979 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1980 &pp->prd_dma, GFP_KERNEL);
1981 if (!pp->prd)
1982 return -ENOMEM;
1983 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1984
1985 ap->private_data = pp;
1986 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1987 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1988 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1989
1990 return 0;
1991}
1992
1993static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1994{
1995 if (qc->tf.protocol != ATA_PROT_NCQ) {
Tejun Heof47451c2010-05-10 21:41:40 +02001996 ata_bmdma_qc_prep(qc);
Kuan Luof140f0f2007-10-15 15:16:53 -04001997 return;
1998 }
1999
2000 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
2001 return;
2002
2003 nv_swncq_fill_sg(qc);
2004}
2005
2006static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
2007{
2008 struct ata_port *ap = qc->ap;
2009 struct scatterlist *sg;
Kuan Luof140f0f2007-10-15 15:16:53 -04002010 struct nv_swncq_port_priv *pp = ap->private_data;
Tejun Heof60d7012010-05-10 21:41:41 +02002011 struct ata_bmdma_prd *prd;
Tejun Heoff2aeb12007-12-05 16:43:11 +09002012 unsigned int si, idx;
Kuan Luof140f0f2007-10-15 15:16:53 -04002013
2014 prd = pp->prd + ATA_MAX_PRD * qc->tag;
2015
2016 idx = 0;
Tejun Heoff2aeb12007-12-05 16:43:11 +09002017 for_each_sg(qc->sg, sg, qc->n_elem, si) {
Kuan Luof140f0f2007-10-15 15:16:53 -04002018 u32 addr, offset;
2019 u32 sg_len, len;
2020
2021 addr = (u32)sg_dma_address(sg);
2022 sg_len = sg_dma_len(sg);
2023
2024 while (sg_len) {
2025 offset = addr & 0xffff;
2026 len = sg_len;
2027 if ((offset + sg_len) > 0x10000)
2028 len = 0x10000 - offset;
2029
2030 prd[idx].addr = cpu_to_le32(addr);
2031 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2032
2033 idx++;
2034 sg_len -= len;
2035 addr += len;
2036 }
2037 }
2038
Tejun Heoff2aeb12007-12-05 16:43:11 +09002039 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
Kuan Luof140f0f2007-10-15 15:16:53 -04002040}
2041
2042static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2043 struct ata_queued_cmd *qc)
2044{
2045 struct nv_swncq_port_priv *pp = ap->private_data;
2046
2047 if (qc == NULL)
2048 return 0;
2049
2050 DPRINTK("Enter\n");
2051
2052 writel((1 << qc->tag), pp->sactive_block);
2053 pp->last_issue_tag = qc->tag;
2054 pp->dhfis_bits &= ~(1 << qc->tag);
2055 pp->dmafis_bits &= ~(1 << qc->tag);
2056 pp->qc_active |= (0x1 << qc->tag);
2057
Tejun Heo5682ed32008-04-07 22:47:16 +09002058 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2059 ap->ops->sff_exec_command(ap, &qc->tf);
Kuan Luof140f0f2007-10-15 15:16:53 -04002060
2061 DPRINTK("Issued tag %u\n", qc->tag);
2062
2063 return 0;
2064}
2065
2066static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2067{
2068 struct ata_port *ap = qc->ap;
2069 struct nv_swncq_port_priv *pp = ap->private_data;
2070
2071 if (qc->tf.protocol != ATA_PROT_NCQ)
Tejun Heo360ff782010-05-10 21:41:42 +02002072 return ata_bmdma_qc_issue(qc);
Kuan Luof140f0f2007-10-15 15:16:53 -04002073
2074 DPRINTK("Enter\n");
2075
2076 if (!pp->qc_active)
2077 nv_swncq_issue_atacmd(ap, qc);
2078 else
2079 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2080
2081 return 0;
2082}
2083
2084static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2085{
2086 u32 serror;
2087 struct ata_eh_info *ehi = &ap->link.eh_info;
2088
2089 ata_ehi_clear_desc(ehi);
2090
2091 /* AHCI needs SError cleared; otherwise, it might lock up */
2092 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2093 sata_scr_write(&ap->link, SCR_ERROR, serror);
2094
2095 /* analyze @irq_stat */
2096 if (fis & NV_SWNCQ_IRQ_ADDED)
2097 ata_ehi_push_desc(ehi, "hot plug");
2098 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2099 ata_ehi_push_desc(ehi, "hot unplug");
2100
2101 ata_ehi_hotplugged(ehi);
2102
2103 /* okay, let's hand over to EH */
2104 ehi->serror |= serror;
2105
2106 ata_port_freeze(ap);
2107}
2108
2109static int nv_swncq_sdbfis(struct ata_port *ap)
2110{
2111 struct ata_queued_cmd *qc;
2112 struct nv_swncq_port_priv *pp = ap->private_data;
2113 struct ata_eh_info *ehi = &ap->link.eh_info;
2114 u32 sactive;
Kuan Luof140f0f2007-10-15 15:16:53 -04002115 u32 done_mask;
Kuan Luof140f0f2007-10-15 15:16:53 -04002116 u8 host_stat;
2117 u8 lack_dhfis = 0;
2118
2119 host_stat = ap->ops->bmdma_status(ap);
2120 if (unlikely(host_stat & ATA_DMA_ERR)) {
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002121 /* error when transferring data to/from memory */
Kuan Luof140f0f2007-10-15 15:16:53 -04002122 ata_ehi_clear_desc(ehi);
2123 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2124 ehi->err_mask |= AC_ERR_HOST_BUS;
Tejun Heocf480622008-01-24 00:05:14 +09002125 ehi->action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04002126 return -EINVAL;
2127 }
2128
Tejun Heo5682ed32008-04-07 22:47:16 +09002129 ap->ops->sff_irq_clear(ap);
Kuan Luof140f0f2007-10-15 15:16:53 -04002130 __ata_bmdma_stop(ap);
2131
2132 sactive = readl(pp->sactive_block);
2133 done_mask = pp->qc_active ^ sactive;
2134
Tejun Heo1aadf5c2010-06-25 15:03:34 +02002135 pp->qc_active &= ~done_mask;
2136 pp->dhfis_bits &= ~done_mask;
2137 pp->dmafis_bits &= ~done_mask;
2138 pp->sdbfis_bits |= done_mask;
2139 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
Kuan Luof140f0f2007-10-15 15:16:53 -04002140
2141 if (!ap->qc_active) {
2142 DPRINTK("over\n");
2143 nv_swncq_pp_reinit(ap);
Tejun Heo752e3862010-06-25 15:02:59 +02002144 return 0;
Kuan Luof140f0f2007-10-15 15:16:53 -04002145 }
2146
2147 if (pp->qc_active & pp->dhfis_bits)
Tejun Heo752e3862010-06-25 15:02:59 +02002148 return 0;
Kuan Luof140f0f2007-10-15 15:16:53 -04002149
2150 if ((pp->ncq_flags & ncq_saw_backout) ||
2151 (pp->qc_active ^ pp->dhfis_bits))
Tejun Heo752e3862010-06-25 15:02:59 +02002152 /* if the controller can't get a device to host register FIS,
Kuan Luof140f0f2007-10-15 15:16:53 -04002153 * The driver needs to reissue the new command.
2154 */
2155 lack_dhfis = 1;
2156
2157 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2158 "SWNCQ:qc_active 0x%X defer_bits %X "
2159 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2160 ap->print_id, ap->qc_active, pp->qc_active,
2161 pp->defer_queue.defer_bits, pp->dhfis_bits,
2162 pp->dmafis_bits, pp->last_issue_tag);
2163
2164 nv_swncq_fis_reinit(ap);
2165
2166 if (lack_dhfis) {
2167 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2168 nv_swncq_issue_atacmd(ap, qc);
Tejun Heo752e3862010-06-25 15:02:59 +02002169 return 0;
Kuan Luof140f0f2007-10-15 15:16:53 -04002170 }
2171
2172 if (pp->defer_queue.defer_bits) {
2173 /* send deferral queue command */
2174 qc = nv_swncq_qc_from_dq(ap);
2175 WARN_ON(qc == NULL);
2176 nv_swncq_issue_atacmd(ap, qc);
2177 }
2178
Tejun Heo752e3862010-06-25 15:02:59 +02002179 return 0;
Kuan Luof140f0f2007-10-15 15:16:53 -04002180}
2181
2182static inline u32 nv_swncq_tag(struct ata_port *ap)
2183{
2184 struct nv_swncq_port_priv *pp = ap->private_data;
2185 u32 tag;
2186
2187 tag = readb(pp->tag_block) >> 2;
2188 return (tag & 0x1f);
2189}
2190
Tejun Heo752e3862010-06-25 15:02:59 +02002191static void nv_swncq_dmafis(struct ata_port *ap)
Kuan Luof140f0f2007-10-15 15:16:53 -04002192{
2193 struct ata_queued_cmd *qc;
2194 unsigned int rw;
2195 u8 dmactl;
2196 u32 tag;
2197 struct nv_swncq_port_priv *pp = ap->private_data;
2198
2199 __ata_bmdma_stop(ap);
2200 tag = nv_swncq_tag(ap);
2201
2202 DPRINTK("dma setup tag 0x%x\n", tag);
2203 qc = ata_qc_from_tag(ap, tag);
2204
2205 if (unlikely(!qc))
Tejun Heo752e3862010-06-25 15:02:59 +02002206 return;
Kuan Luof140f0f2007-10-15 15:16:53 -04002207
2208 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2209
2210 /* load PRD table addr. */
2211 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2212 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2213
2214 /* specify data direction, triple-check start bit is clear */
2215 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2216 dmactl &= ~ATA_DMA_WR;
2217 if (!rw)
2218 dmactl |= ATA_DMA_WR;
2219
2220 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
Kuan Luof140f0f2007-10-15 15:16:53 -04002221}
2222
2223static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2224{
2225 struct nv_swncq_port_priv *pp = ap->private_data;
2226 struct ata_queued_cmd *qc;
2227 struct ata_eh_info *ehi = &ap->link.eh_info;
2228 u32 serror;
2229 u8 ata_stat;
Kuan Luof140f0f2007-10-15 15:16:53 -04002230
Tejun Heo5682ed32008-04-07 22:47:16 +09002231 ata_stat = ap->ops->sff_check_status(ap);
Kuan Luof140f0f2007-10-15 15:16:53 -04002232 nv_swncq_irq_clear(ap, fis);
2233 if (!fis)
2234 return;
2235
2236 if (ap->pflags & ATA_PFLAG_FROZEN)
2237 return;
2238
2239 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2240 nv_swncq_hotplug(ap, fis);
2241 return;
2242 }
2243
2244 if (!pp->qc_active)
2245 return;
2246
Tejun Heo82ef04f2008-07-31 17:02:40 +09002247 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
Kuan Luof140f0f2007-10-15 15:16:53 -04002248 return;
Tejun Heo82ef04f2008-07-31 17:02:40 +09002249 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
Kuan Luof140f0f2007-10-15 15:16:53 -04002250
2251 if (ata_stat & ATA_ERR) {
2252 ata_ehi_clear_desc(ehi);
2253 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2254 ehi->err_mask |= AC_ERR_DEV;
2255 ehi->serror |= serror;
Tejun Heocf480622008-01-24 00:05:14 +09002256 ehi->action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04002257 ata_port_freeze(ap);
2258 return;
2259 }
2260
2261 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2262 /* If the IRQ is backout, driver must issue
2263 * the new command again some time later.
2264 */
2265 pp->ncq_flags |= ncq_saw_backout;
2266 }
2267
2268 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2269 pp->ncq_flags |= ncq_saw_sdb;
2270 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2271 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2272 ap->print_id, pp->qc_active, pp->dhfis_bits,
2273 pp->dmafis_bits, readl(pp->sactive_block));
Tejun Heo752e3862010-06-25 15:02:59 +02002274 if (nv_swncq_sdbfis(ap) < 0)
Kuan Luof140f0f2007-10-15 15:16:53 -04002275 goto irq_error;
2276 }
2277
2278 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2279 /* The interrupt indicates the new command
2280 * was transmitted correctly to the drive.
2281 */
2282 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2283 pp->ncq_flags |= ncq_saw_d2h;
2284 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2285 ata_ehi_push_desc(ehi, "illegal fis transaction");
2286 ehi->err_mask |= AC_ERR_HSM;
Tejun Heocf480622008-01-24 00:05:14 +09002287 ehi->action |= ATA_EH_RESET;
Kuan Luof140f0f2007-10-15 15:16:53 -04002288 goto irq_error;
2289 }
2290
2291 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2292 !(pp->ncq_flags & ncq_saw_dmas)) {
Tejun Heo5682ed32008-04-07 22:47:16 +09002293 ata_stat = ap->ops->sff_check_status(ap);
Kuan Luof140f0f2007-10-15 15:16:53 -04002294 if (ata_stat & ATA_BUSY)
2295 goto irq_exit;
2296
2297 if (pp->defer_queue.defer_bits) {
2298 DPRINTK("send next command\n");
2299 qc = nv_swncq_qc_from_dq(ap);
2300 nv_swncq_issue_atacmd(ap, qc);
2301 }
2302 }
2303 }
2304
2305 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2306 /* program the dma controller with appropriate PRD buffers
2307 * and start the DMA transfer for requested command.
2308 */
2309 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2310 pp->ncq_flags |= ncq_saw_dmas;
Tejun Heo752e3862010-06-25 15:02:59 +02002311 nv_swncq_dmafis(ap);
Kuan Luof140f0f2007-10-15 15:16:53 -04002312 }
2313
2314irq_exit:
2315 return;
2316irq_error:
2317 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2318 ata_port_freeze(ap);
2319 return;
2320}
2321
2322static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2323{
2324 struct ata_host *host = dev_instance;
2325 unsigned int i;
2326 unsigned int handled = 0;
2327 unsigned long flags;
2328 u32 irq_stat;
2329
2330 spin_lock_irqsave(&host->lock, flags);
2331
2332 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2333
2334 for (i = 0; i < host->n_ports; i++) {
2335 struct ata_port *ap = host->ports[i];
2336
Tejun Heo3e4ec342010-05-10 21:41:30 +02002337 if (ap->link.sactive) {
2338 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2339 handled = 1;
2340 } else {
2341 if (irq_stat) /* reserve Hotplug */
2342 nv_swncq_irq_clear(ap, 0xfff0);
Kuan Luof140f0f2007-10-15 15:16:53 -04002343
Tejun Heo3e4ec342010-05-10 21:41:30 +02002344 handled += nv_host_intr(ap, (u8)irq_stat);
Kuan Luof140f0f2007-10-15 15:16:53 -04002345 }
2346 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2347 }
2348
2349 spin_unlock_irqrestore(&host->lock, flags);
2350
2351 return IRQ_RETVAL(handled);
2352}
2353
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002354static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002355{
Tejun Heo1626aeb2007-05-04 12:43:58 +02002356 const struct ata_port_info *ppi[] = { NULL, NULL };
Tejun Heo95947192008-03-25 12:22:49 +09002357 struct nv_pi_priv *ipriv;
Tejun Heo9a829cc2007-04-17 23:44:08 +09002358 struct ata_host *host;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002359 struct nv_host_priv *hpriv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002360 int rc;
2361 u32 bar;
Tejun Heo0d5ff562007-02-01 15:06:36 +09002362 void __iomem *base;
Robert Hancockfbbb2622006-10-27 19:08:41 -07002363 unsigned long type = ent->driver_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002364
2365 // Make sure this is a SATA controller by counting the number of bars
2366 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2367 // it's an IDE controller and we ignore it.
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002368 for (bar = 0; bar < 6; bar++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002369 if (pci_resource_start(pdev, bar) == 0)
2370 return -ENODEV;
2371
Joe Perches06296a12011-04-15 15:52:00 -07002372 ata_print_version_once(&pdev->dev, DRV_VERSION);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373
Tejun Heo24dc5f32007-01-20 16:00:28 +09002374 rc = pcim_enable_device(pdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002375 if (rc)
Tejun Heo24dc5f32007-01-20 16:00:28 +09002376 return rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002377
Tejun Heo9a829cc2007-04-17 23:44:08 +09002378 /* determine type and allocate host */
Kuan Luof140f0f2007-10-15 15:16:53 -04002379 if (type == CK804 && adma_enabled) {
Joe Perchesa44fec12011-04-15 15:51:58 -07002380 dev_notice(&pdev->dev, "Using ADMA mode\n");
Robert Hancockfbbb2622006-10-27 19:08:41 -07002381 type = ADMA;
Tejun Heo2d775702009-01-25 11:29:38 +09002382 } else if (type == MCP5x && swncq_enabled) {
Joe Perchesa44fec12011-04-15 15:51:58 -07002383 dev_notice(&pdev->dev, "Using SWNCQ mode\n");
Tejun Heo2d775702009-01-25 11:29:38 +09002384 type = SWNCQ;
Jeff Garzik360737a2007-10-29 06:49:24 -04002385 }
2386
Tejun Heo1626aeb2007-05-04 12:43:58 +02002387 ppi[0] = &nv_port_info[type];
Tejun Heo95947192008-03-25 12:22:49 +09002388 ipriv = ppi[0]->private_data;
Tejun Heo1c5afdf2010-05-19 22:10:22 +02002389 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host);
Tejun Heo9a829cc2007-04-17 23:44:08 +09002390 if (rc)
2391 return rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392
Tejun Heo24dc5f32007-01-20 16:00:28 +09002393 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002394 if (!hpriv)
Tejun Heo24dc5f32007-01-20 16:00:28 +09002395 return -ENOMEM;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002396 hpriv->type = type;
Tejun Heo9a829cc2007-04-17 23:44:08 +09002397 host->private_data = hpriv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398
Tejun Heo9a829cc2007-04-17 23:44:08 +09002399 /* request and iomap NV_MMIO_BAR */
2400 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2401 if (rc)
2402 return rc;
2403
2404 /* configure SCR access */
2405 base = host->iomap[NV_MMIO_BAR];
2406 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2407 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
Jeff Garzik02cbd922006-03-22 23:59:46 -05002408
Tejun Heoada364e2006-06-17 15:49:56 +09002409 /* enable SATA space for CK804 */
Robert Hancockfbbb2622006-10-27 19:08:41 -07002410 if (type >= CK804) {
Tejun Heoada364e2006-06-17 15:49:56 +09002411 u8 regval;
2412
2413 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2414 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2415 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2416 }
2417
Tejun Heo9a829cc2007-04-17 23:44:08 +09002418 /* init ADMA */
Robert Hancockfbbb2622006-10-27 19:08:41 -07002419 if (type == ADMA) {
Tejun Heo9a829cc2007-04-17 23:44:08 +09002420 rc = nv_adma_host_init(host);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002421 if (rc)
Tejun Heo24dc5f32007-01-20 16:00:28 +09002422 return rc;
Jeff Garzik360737a2007-10-29 06:49:24 -04002423 } else if (type == SWNCQ)
Kuan Luof140f0f2007-10-15 15:16:53 -04002424 nv_swncq_host_init(host);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002425
Tony Vroon51c89492009-08-06 00:50:09 +01002426 if (msi_enabled) {
Joe Perchesa44fec12011-04-15 15:51:58 -07002427 dev_notice(&pdev->dev, "Using MSI\n");
Tony Vroon51c89492009-08-06 00:50:09 +01002428 pci_enable_msi(pdev);
2429 }
2430
Tejun Heo9a829cc2007-04-17 23:44:08 +09002431 pci_set_master(pdev);
Tejun Heo95cc2c72010-05-14 11:48:50 +02002432 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002433}
2434
Tejun Heo438ac6d2007-03-02 17:31:26 +09002435#ifdef CONFIG_PM
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002436static int nv_pci_device_resume(struct pci_dev *pdev)
2437{
2438 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2439 struct nv_host_priv *hpriv = host->private_data;
Robert Hancockce053fa2007-02-05 16:26:04 -08002440 int rc;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002441
Robert Hancockce053fa2007-02-05 16:26:04 -08002442 rc = ata_pci_device_do_resume(pdev);
Jeff Garzikb4479162007-10-25 20:47:30 -04002443 if (rc)
Robert Hancockce053fa2007-02-05 16:26:04 -08002444 return rc;
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002445
2446 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
Jeff Garzikb4479162007-10-25 20:47:30 -04002447 if (hpriv->type >= CK804) {
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002448 u8 regval;
2449
2450 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2451 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2452 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2453 }
Jeff Garzikb4479162007-10-25 20:47:30 -04002454 if (hpriv->type == ADMA) {
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002455 u32 tmp32;
2456 struct nv_adma_port_priv *pp;
2457 /* enable/disable ADMA on the ports appropriately */
2458 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2459
2460 pp = host->ports[0]->private_data;
Jeff Garzikb4479162007-10-25 20:47:30 -04002461 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002462 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002463 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002464 else
2465 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002466 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002467 pp = host->ports[1]->private_data;
Jeff Garzikb4479162007-10-25 20:47:30 -04002468 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002469 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002470 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002471 else
2472 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
Jeff Garzik5796d1c2007-10-26 00:03:37 -04002473 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002474
2475 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2476 }
2477 }
2478
2479 ata_host_resume(host);
2480
2481 return 0;
2482}
Tejun Heo438ac6d2007-03-02 17:31:26 +09002483#endif
Robert Hancockcdf56bc2007-01-03 18:13:57 -06002484
Jeff Garzikcca39742006-08-24 03:19:22 -04002485static void nv_ck804_host_stop(struct ata_host *host)
Tejun Heoada364e2006-06-17 15:49:56 +09002486{
Jeff Garzikcca39742006-08-24 03:19:22 -04002487 struct pci_dev *pdev = to_pci_dev(host->dev);
Tejun Heoada364e2006-06-17 15:49:56 +09002488 u8 regval;
2489
2490 /* disable SATA space for CK804 */
2491 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2492 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2493 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
Tejun Heoada364e2006-06-17 15:49:56 +09002494}
2495
Robert Hancockfbbb2622006-10-27 19:08:41 -07002496static void nv_adma_host_stop(struct ata_host *host)
2497{
2498 struct pci_dev *pdev = to_pci_dev(host->dev);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002499 u32 tmp32;
2500
Robert Hancockfbbb2622006-10-27 19:08:41 -07002501 /* disable ADMA on the ports */
2502 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2503 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2504 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2505 NV_MCP_SATA_CFG_20_PORT1_EN |
2506 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2507
2508 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2509
2510 nv_ck804_host_stop(host);
2511}
2512
Linus Torvalds1da177e2005-04-16 15:20:36 -07002513static int __init nv_init(void)
2514{
Pavel Roskinb7887192006-08-10 18:13:18 +09002515 return pci_register_driver(&nv_pci_driver);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002516}
2517
2518static void __exit nv_exit(void)
2519{
2520 pci_unregister_driver(&nv_pci_driver);
2521}
2522
2523module_init(nv_init);
2524module_exit(nv_exit);
Robert Hancockfbbb2622006-10-27 19:08:41 -07002525module_param_named(adma, adma_enabled, bool, 0444);
Brandon Ehle55f784c2009-03-01 00:02:49 -08002526MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
Kuan Luof140f0f2007-10-15 15:16:53 -04002527module_param_named(swncq, swncq_enabled, bool, 0444);
Zoltan Boszormenyid21279f2008-03-28 14:33:46 -07002528MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
Tony Vroon51c89492009-08-06 00:50:09 +01002529module_param_named(msi, msi_enabled, bool, 0444);
2530MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");
Kuan Luof140f0f2007-10-15 15:16:53 -04002531