blob: 391a874e41330bb6a2cbf347070c787b671f0f43 [file] [log] [blame]
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001/*
2 * NVM Express device driver
3 * Copyright (c) 2011, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 */
18
19#include <linux/nvme.h>
20#include <linux/bio.h>
Matthew Wilcox8de05532011-05-12 13:50:28 -040021#include <linux/bitops.h>
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050022#include <linux/blkdev.h>
Matthew Wilcoxfd63e9ce2011-05-06 08:37:54 -040023#include <linux/delay.h>
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050024#include <linux/errno.h>
25#include <linux/fs.h>
26#include <linux/genhd.h>
Matthew Wilcox5aff9382011-05-06 08:45:47 -040027#include <linux/idr.h>
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050028#include <linux/init.h>
29#include <linux/interrupt.h>
30#include <linux/io.h>
31#include <linux/kdev_t.h>
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -050032#include <linux/kthread.h>
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050033#include <linux/kernel.h>
34#include <linux/mm.h>
35#include <linux/module.h>
36#include <linux/moduleparam.h>
37#include <linux/pci.h>
Matthew Wilcoxbe7b6272011-02-06 07:53:23 -050038#include <linux/poison.h>
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050039#include <linux/sched.h>
40#include <linux/slab.h>
41#include <linux/types.h>
Vishal Verma5d0f6132013-03-04 18:40:58 -070042#include <scsi/sg.h>
Hitoshi Mitake797a7962012-02-07 11:45:33 +090043#include <asm-generic/io-64-nonatomic-lo-hi.h>
44
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050045#define NVME_Q_DEPTH 1024
46#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
47#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
48#define NVME_MINORS 64
Matthew Wilcoxe85248e2011-02-06 18:30:16 -050049#define ADMIN_TIMEOUT (60 * HZ)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050050
51static int nvme_major;
52module_param(nvme_major, int, 0);
53
Matthew Wilcox58ffacb2011-02-06 07:28:06 -050054static int use_threaded_interrupts;
55module_param(use_threaded_interrupts, int, 0);
56
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -050057static DEFINE_SPINLOCK(dev_list_lock);
58static LIST_HEAD(dev_list);
59static struct task_struct *nvme_thread;
60
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050061/*
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050062 * An NVM Express queue. Each device has at least two (one for admin
63 * commands and one for I/O commands).
64 */
65struct nvme_queue {
66 struct device *q_dmadev;
Matthew Wilcox091b6092011-02-10 09:56:01 -050067 struct nvme_dev *dev;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050068 spinlock_t q_lock;
69 struct nvme_command *sq_cmds;
70 volatile struct nvme_completion *cqes;
71 dma_addr_t sq_dma_addr;
72 dma_addr_t cq_dma_addr;
73 wait_queue_head_t sq_full;
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -050074 wait_queue_t sq_cong_wait;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050075 struct bio_list sq_cong;
76 u32 __iomem *q_db;
77 u16 q_depth;
78 u16 cq_vector;
79 u16 sq_head;
80 u16 sq_tail;
81 u16 cq_head;
Matthew Wilcox82123462011-01-20 13:24:06 -050082 u16 cq_phase;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050083 unsigned long cmdid_data[];
84};
85
86/*
87 * Check we didin't inadvertently grow the command struct
88 */
89static inline void _nvme_check_size(void)
90{
91 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
92 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
93 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
94 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
95 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
Vishal Vermaf8ebf842013-03-27 07:13:41 -040096 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -050097 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
98 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
99 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
100 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
Keith Busch6ecec742012-09-26 12:49:27 -0600101 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500102}
103
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500104typedef void (*nvme_completion_fn)(struct nvme_dev *, void *,
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400105 struct nvme_completion *);
106
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500107struct nvme_cmd_info {
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400108 nvme_completion_fn fn;
109 void *ctx;
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500110 unsigned long timeout;
111};
112
113static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq)
114{
115 return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)];
116}
117
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500118/**
Matthew Wilcox714a7a22011-03-16 16:28:24 -0400119 * alloc_cmdid() - Allocate a Command ID
120 * @nvmeq: The queue that will be used for this command
121 * @ctx: A pointer that will be passed to the handler
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400122 * @handler: The function to call on completion
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500123 *
124 * Allocate a Command ID for a queue. The data passed in will
125 * be passed to the completion handler. This is implemented by using
126 * the bottom two bits of the ctx pointer to store the handler ID.
127 * Passing in a pointer that's not 4-byte aligned will cause a BUG.
128 * We can change this if it becomes a problem.
Matthew Wilcox184d2942011-05-11 21:36:38 -0400129 *
130 * May be called with local interrupts disabled and the q_lock held,
131 * or with interrupts enabled and no locks held.
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500132 */
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400133static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx,
134 nvme_completion_fn handler, unsigned timeout)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500135{
Matthew Wilcoxe6d15f72011-02-24 08:49:41 -0500136 int depth = nvmeq->q_depth - 1;
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500137 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500138 int cmdid;
139
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500140 do {
141 cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth);
142 if (cmdid >= depth)
143 return -EBUSY;
144 } while (test_and_set_bit(cmdid, nvmeq->cmdid_data));
145
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400146 info[cmdid].fn = handler;
147 info[cmdid].ctx = ctx;
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500148 info[cmdid].timeout = jiffies + timeout;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500149 return cmdid;
150}
151
152static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400153 nvme_completion_fn handler, unsigned timeout)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500154{
155 int cmdid;
156 wait_event_killable(nvmeq->sq_full,
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500157 (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500158 return (cmdid < 0) ? -EINTR : cmdid;
159}
160
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400161/* Special values must be less than 0x1000 */
162#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA)
Matthew Wilcoxd2d87032011-02-07 15:55:59 -0500163#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE)
164#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE)
165#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE)
Matthew Wilcox00df5cb2011-02-22 14:18:30 -0500166#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE)
Matthew Wilcoxbe7b6272011-02-06 07:53:23 -0500167
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500168static void special_completion(struct nvme_dev *dev, void *ctx,
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400169 struct nvme_completion *cqe)
170{
171 if (ctx == CMD_CTX_CANCELLED)
172 return;
173 if (ctx == CMD_CTX_FLUSH)
174 return;
175 if (ctx == CMD_CTX_COMPLETED) {
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500176 dev_warn(&dev->pci_dev->dev,
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400177 "completed id %d twice on queue %d\n",
178 cqe->command_id, le16_to_cpup(&cqe->sq_id));
179 return;
180 }
181 if (ctx == CMD_CTX_INVALID) {
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500182 dev_warn(&dev->pci_dev->dev,
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400183 "invalid id %d completed on queue %d\n",
184 cqe->command_id, le16_to_cpup(&cqe->sq_id));
185 return;
186 }
187
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500188 dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx);
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400189}
190
Matthew Wilcox184d2942011-05-11 21:36:38 -0400191/*
192 * Called with local interrupts disabled and the q_lock held. May not sleep.
193 */
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400194static void *free_cmdid(struct nvme_queue *nvmeq, int cmdid,
195 nvme_completion_fn *fn)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500196{
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400197 void *ctx;
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500198 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500199
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400200 if (cmdid >= nvmeq->q_depth) {
201 *fn = special_completion;
Matthew Wilcox48e3d392011-02-06 08:51:15 -0500202 return CMD_CTX_INVALID;
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400203 }
Keith Busch859361a2012-08-02 14:05:59 -0600204 if (fn)
205 *fn = info[cmdid].fn;
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400206 ctx = info[cmdid].ctx;
207 info[cmdid].fn = special_completion;
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500208 info[cmdid].ctx = CMD_CTX_COMPLETED;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500209 clear_bit(cmdid, nvmeq->cmdid_data);
210 wake_up(&nvmeq->sq_full);
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400211 return ctx;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500212}
213
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400214static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,
215 nvme_completion_fn *fn)
Matthew Wilcox3c0cf132011-02-04 16:03:56 -0500216{
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400217 void *ctx;
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500218 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400219 if (fn)
220 *fn = info[cmdid].fn;
221 ctx = info[cmdid].ctx;
222 info[cmdid].fn = special_completion;
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500223 info[cmdid].ctx = CMD_CTX_CANCELLED;
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400224 return ctx;
Matthew Wilcox3c0cf132011-02-04 16:03:56 -0500225}
226
Vishal Verma5d0f6132013-03-04 18:40:58 -0700227struct nvme_queue *get_nvmeq(struct nvme_dev *dev)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500228{
Matthew Wilcox040a93b2011-12-20 11:04:12 -0500229 return dev->queues[get_cpu() + 1];
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500230}
231
Vishal Verma5d0f6132013-03-04 18:40:58 -0700232void put_nvmeq(struct nvme_queue *nvmeq)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500233{
Matthew Wilcox1b234842011-01-20 13:01:49 -0500234 put_cpu();
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500235}
236
237/**
Matthew Wilcox714a7a22011-03-16 16:28:24 -0400238 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500239 * @nvmeq: The queue to use
240 * @cmd: The command to send
241 *
242 * Safe to use from interrupt context
243 */
244static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
245{
246 unsigned long flags;
247 u16 tail;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500248 spin_lock_irqsave(&nvmeq->q_lock, flags);
249 tail = nvmeq->sq_tail;
250 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500251 if (++tail == nvmeq->q_depth)
252 tail = 0;
Matthew Wilcox75478812011-02-16 09:59:59 -0500253 writel(tail, nvmeq->q_db);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500254 nvmeq->sq_tail = tail;
255 spin_unlock_irqrestore(&nvmeq->q_lock, flags);
256
257 return 0;
258}
259
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500260static __le64 **iod_list(struct nvme_iod *iod)
261{
262 return ((void *)iod) + iod->offset;
263}
264
265/*
266 * Will slightly overestimate the number of pages needed. This is OK
267 * as it only leads to a small amount of wasted memory for the lifetime of
268 * the I/O.
269 */
270static int nvme_npages(unsigned size)
271{
272 unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE);
273 return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
274}
275
276static struct nvme_iod *
277nvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp)
278{
279 struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
280 sizeof(__le64 *) * nvme_npages(nbytes) +
281 sizeof(struct scatterlist) * nseg, gfp);
282
283 if (iod) {
284 iod->offset = offsetof(struct nvme_iod, sg[nseg]);
285 iod->npages = -1;
286 iod->length = nbytes;
Keith Busch2b196032012-11-06 11:59:23 -0700287 iod->nents = 0;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500288 }
289
290 return iod;
291}
292
Vishal Verma5d0f6132013-03-04 18:40:58 -0700293void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500294{
295 const int last_prp = PAGE_SIZE / 8 - 1;
296 int i;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500297 __le64 **list = iod_list(iod);
298 dma_addr_t prp_dma = iod->first_dma;
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500299
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500300 if (iod->npages == 0)
301 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
302 for (i = 0; i < iod->npages; i++) {
303 __le64 *prp_list = list[i];
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500304 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
Matthew Wilcox091b6092011-02-10 09:56:01 -0500305 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500306 prp_dma = next_prp_dma;
307 }
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500308 kfree(iod);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500309}
310
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500311static void requeue_bio(struct nvme_dev *dev, struct bio *bio)
312{
313 struct nvme_queue *nvmeq = get_nvmeq(dev);
314 if (bio_list_empty(&nvmeq->sq_cong))
315 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
316 bio_list_add(&nvmeq->sq_cong, bio);
317 put_nvmeq(nvmeq);
318 wake_up_process(nvme_thread);
319}
320
321static void bio_completion(struct nvme_dev *dev, void *ctx,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500322 struct nvme_completion *cqe)
323{
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500324 struct nvme_iod *iod = ctx;
325 struct bio *bio = iod->private;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500326 u16 status = le16_to_cpup(&cqe->status) >> 1;
327
Keith Busch2b196032012-11-06 11:59:23 -0700328 if (iod->nents)
329 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500330 bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500331 nvme_free_iod(dev, iod);
Matthew Wilcox09a58f52011-04-28 23:09:09 -0700332 if (status) {
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500333 bio_endio(bio, -EIO);
Matthew Wilcox09a58f52011-04-28 23:09:09 -0700334 } else if (bio->bi_vcnt > bio->bi_idx) {
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500335 requeue_bio(dev, bio);
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500336 } else {
337 bio_endio(bio, 0);
338 }
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500339}
340
Matthew Wilcox184d2942011-05-11 21:36:38 -0400341/* length is in bytes. gfp flags indicates whether we may sleep. */
Vishal Verma5d0f6132013-03-04 18:40:58 -0700342int nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd,
343 struct nvme_iod *iod, int total_len, gfp_t gfp)
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500344{
Matthew Wilcox99802a72011-02-10 10:30:34 -0500345 struct dma_pool *pool;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500346 int length = total_len;
347 struct scatterlist *sg = iod->sg;
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500348 int dma_len = sg_dma_len(sg);
349 u64 dma_addr = sg_dma_address(sg);
350 int offset = offset_in_page(dma_addr);
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500351 __le64 *prp_list;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500352 __le64 **list = iod_list(iod);
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500353 dma_addr_t prp_dma;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500354 int nprps, i;
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500355
356 cmd->prp1 = cpu_to_le64(dma_addr);
357 length -= (PAGE_SIZE - offset);
358 if (length <= 0)
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500359 return total_len;
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500360
361 dma_len -= (PAGE_SIZE - offset);
362 if (dma_len) {
363 dma_addr += (PAGE_SIZE - offset);
364 } else {
365 sg = sg_next(sg);
366 dma_addr = sg_dma_address(sg);
367 dma_len = sg_dma_len(sg);
368 }
369
370 if (length <= PAGE_SIZE) {
371 cmd->prp2 = cpu_to_le64(dma_addr);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500372 return total_len;
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500373 }
374
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500375 nprps = DIV_ROUND_UP(length, PAGE_SIZE);
Matthew Wilcox99802a72011-02-10 10:30:34 -0500376 if (nprps <= (256 / 8)) {
377 pool = dev->prp_small_pool;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500378 iod->npages = 0;
Matthew Wilcox99802a72011-02-10 10:30:34 -0500379 } else {
380 pool = dev->prp_page_pool;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500381 iod->npages = 1;
Matthew Wilcox99802a72011-02-10 10:30:34 -0500382 }
383
Matthew Wilcoxb77954c2011-05-12 13:51:41 -0400384 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
385 if (!prp_list) {
386 cmd->prp2 = cpu_to_le64(dma_addr);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500387 iod->npages = -1;
388 return (total_len - length) + PAGE_SIZE;
Matthew Wilcoxb77954c2011-05-12 13:51:41 -0400389 }
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500390 list[0] = prp_list;
391 iod->first_dma = prp_dma;
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500392 cmd->prp2 = cpu_to_le64(prp_dma);
393 i = 0;
394 for (;;) {
Matthew Wilcox7523d832011-03-16 16:43:40 -0400395 if (i == PAGE_SIZE / 8) {
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500396 __le64 *old_prp_list = prp_list;
Matthew Wilcoxb77954c2011-05-12 13:51:41 -0400397 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500398 if (!prp_list)
399 return total_len - length;
400 list[iod->npages++] = prp_list;
Matthew Wilcox7523d832011-03-16 16:43:40 -0400401 prp_list[0] = old_prp_list[i - 1];
402 old_prp_list[i - 1] = cpu_to_le64(prp_dma);
403 i = 1;
Shane Michael Matthewse025344c2011-02-10 08:51:24 -0500404 }
405 prp_list[i++] = cpu_to_le64(dma_addr);
406 dma_len -= PAGE_SIZE;
407 dma_addr += PAGE_SIZE;
408 length -= PAGE_SIZE;
409 if (length <= 0)
410 break;
411 if (dma_len > 0)
412 continue;
413 BUG_ON(dma_len < 0);
414 sg = sg_next(sg);
415 dma_addr = sg_dma_address(sg);
416 dma_len = sg_dma_len(sg);
417 }
418
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500419 return total_len;
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500420}
421
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500422/* NVMe scatterlists require no holes in the virtual address */
423#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \
424 (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE))
425
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500426static int nvme_map_bio(struct device *dev, struct nvme_iod *iod,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500427 struct bio *bio, enum dma_data_direction dma_dir, int psegs)
428{
Matthew Wilcox76830842011-02-10 13:55:39 -0500429 struct bio_vec *bvec, *bvprv = NULL;
430 struct scatterlist *sg = NULL;
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500431 int i, old_idx, length = 0, nsegs = 0;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500432
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500433 sg_init_table(iod->sg, psegs);
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500434 old_idx = bio->bi_idx;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500435 bio_for_each_segment(bvec, bio, i) {
Matthew Wilcox76830842011-02-10 13:55:39 -0500436 if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) {
437 sg->length += bvec->bv_len;
438 } else {
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500439 if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec))
440 break;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500441 sg = sg ? sg + 1 : iod->sg;
Matthew Wilcox76830842011-02-10 13:55:39 -0500442 sg_set_page(sg, bvec->bv_page, bvec->bv_len,
443 bvec->bv_offset);
444 nsegs++;
445 }
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500446 length += bvec->bv_len;
Matthew Wilcox76830842011-02-10 13:55:39 -0500447 bvprv = bvec;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500448 }
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500449 bio->bi_idx = i;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500450 iod->nents = nsegs;
Matthew Wilcox76830842011-02-10 13:55:39 -0500451 sg_mark_end(sg);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500452 if (dma_map_sg(dev, iod->sg, iod->nents, dma_dir) == 0) {
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500453 bio->bi_idx = old_idx;
454 return -ENOMEM;
455 }
456 return length;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500457}
458
Keith Busch0e5e4f02012-11-09 16:33:05 -0700459/*
460 * We reuse the small pool to allocate the 16-byte range here as it is not
461 * worth having a special pool for these or additional cases to handle freeing
462 * the iod.
463 */
464static int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
465 struct bio *bio, struct nvme_iod *iod, int cmdid)
466{
467 struct nvme_dsm_range *range;
468 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
469
470 range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
471 &iod->first_dma);
472 if (!range)
473 return -ENOMEM;
474
475 iod_list(iod)[0] = (__le64 *)range;
476 iod->npages = 0;
477
478 range->cattr = cpu_to_le32(0);
479 range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift);
Matthew Wilcox063cc6d2013-03-27 21:28:22 -0400480 range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
Keith Busch0e5e4f02012-11-09 16:33:05 -0700481
482 memset(cmnd, 0, sizeof(*cmnd));
483 cmnd->dsm.opcode = nvme_cmd_dsm;
484 cmnd->dsm.command_id = cmdid;
485 cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
486 cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
487 cmnd->dsm.nr = 0;
488 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
489
490 if (++nvmeq->sq_tail == nvmeq->q_depth)
491 nvmeq->sq_tail = 0;
492 writel(nvmeq->sq_tail, nvmeq->q_db);
493
494 return 0;
495}
496
Matthew Wilcox00df5cb2011-02-22 14:18:30 -0500497static int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
498 int cmdid)
499{
500 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
501
502 memset(cmnd, 0, sizeof(*cmnd));
503 cmnd->common.opcode = nvme_cmd_flush;
504 cmnd->common.command_id = cmdid;
505 cmnd->common.nsid = cpu_to_le32(ns->ns_id);
506
507 if (++nvmeq->sq_tail == nvmeq->q_depth)
508 nvmeq->sq_tail = 0;
509 writel(nvmeq->sq_tail, nvmeq->q_db);
510
511 return 0;
512}
513
Vishal Verma5d0f6132013-03-04 18:40:58 -0700514int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns)
Matthew Wilcox00df5cb2011-02-22 14:18:30 -0500515{
516 int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH,
Matthew Wilcoxff976d72011-12-20 13:53:01 -0500517 special_completion, NVME_IO_TIMEOUT);
Matthew Wilcox00df5cb2011-02-22 14:18:30 -0500518 if (unlikely(cmdid < 0))
519 return cmdid;
520
521 return nvme_submit_flush(nvmeq, ns, cmdid);
522}
523
Matthew Wilcox184d2942011-05-11 21:36:38 -0400524/*
525 * Called with local interrupts disabled and the q_lock held. May not sleep.
526 */
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500527static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,
528 struct bio *bio)
529{
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500530 struct nvme_command *cmnd;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500531 struct nvme_iod *iod;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500532 enum dma_data_direction dma_dir;
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500533 int cmdid, length, result = -ENOMEM;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500534 u16 control;
535 u32 dsmgmt;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500536 int psegs = bio_phys_segments(ns->queue, bio);
537
Matthew Wilcox00df5cb2011-02-22 14:18:30 -0500538 if ((bio->bi_rw & REQ_FLUSH) && psegs) {
539 result = nvme_submit_flush_data(nvmeq, ns);
540 if (result)
541 return result;
542 }
543
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500544 iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC);
545 if (!iod)
Matthew Wilcoxeeee3222011-02-14 15:55:33 -0500546 goto nomem;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500547 iod->private = bio;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500548
Matthew Wilcoxeeee3222011-02-14 15:55:33 -0500549 result = -EBUSY;
Matthew Wilcoxff976d72011-12-20 13:53:01 -0500550 cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500551 if (unlikely(cmdid < 0))
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500552 goto free_iod;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500553
Keith Busch0e5e4f02012-11-09 16:33:05 -0700554 if (bio->bi_rw & REQ_DISCARD) {
555 result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid);
556 if (result)
557 goto free_cmdid;
558 return result;
559 }
Matthew Wilcox00df5cb2011-02-22 14:18:30 -0500560 if ((bio->bi_rw & REQ_FLUSH) && !psegs)
561 return nvme_submit_flush(nvmeq, ns, cmdid);
562
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500563 control = 0;
564 if (bio->bi_rw & REQ_FUA)
565 control |= NVME_RW_FUA;
566 if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD))
567 control |= NVME_RW_LR;
568
569 dsmgmt = 0;
570 if (bio->bi_rw & REQ_RAHEAD)
571 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
572
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500573 cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500574
Matthew Wilcoxb8deb622011-01-26 10:08:25 -0500575 memset(cmnd, 0, sizeof(*cmnd));
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500576 if (bio_data_dir(bio)) {
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500577 cmnd->rw.opcode = nvme_cmd_write;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500578 dma_dir = DMA_TO_DEVICE;
579 } else {
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500580 cmnd->rw.opcode = nvme_cmd_read;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500581 dma_dir = DMA_FROM_DEVICE;
582 }
583
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500584 result = nvme_map_bio(nvmeq->q_dmadev, iod, bio, dma_dir, psegs);
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500585 if (result < 0)
Keith Busch859361a2012-08-02 14:05:59 -0600586 goto free_cmdid;
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500587 length = result;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500588
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500589 cmnd->rw.command_id = cmdid;
590 cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500591 length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length,
592 GFP_ATOMIC);
Matthew Wilcox063cc6d2013-03-27 21:28:22 -0400593 cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
Matthew Wilcox1ad2f892011-02-23 15:20:00 -0500594 cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1);
Matthew Wilcoxff22b542011-01-26 10:02:29 -0500595 cmnd->rw.control = cpu_to_le16(control);
596 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500597
Matthew Wilcoxd8ee9d62011-02-24 08:46:00 -0500598 bio->bi_sector += length >> 9;
599
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500600 if (++nvmeq->sq_tail == nvmeq->q_depth)
601 nvmeq->sq_tail = 0;
Matthew Wilcox75478812011-02-16 09:59:59 -0500602 writel(nvmeq->sq_tail, nvmeq->q_db);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500603
Matthew Wilcox1974b1a2011-02-10 12:01:09 -0500604 return 0;
605
Keith Busch859361a2012-08-02 14:05:59 -0600606 free_cmdid:
607 free_cmdid(nvmeq, cmdid, NULL);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -0500608 free_iod:
609 nvme_free_iod(nvmeq->dev, iod);
Matthew Wilcoxeeee3222011-02-14 15:55:33 -0500610 nomem:
611 return result;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500612}
613
Linus Torvalds93c3d652012-01-18 15:41:27 -0800614static void nvme_make_request(struct request_queue *q, struct bio *bio)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500615{
616 struct nvme_ns *ns = q->queuedata;
Matthew Wilcox040a93b2011-12-20 11:04:12 -0500617 struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
Matthew Wilcoxeeee3222011-02-14 15:55:33 -0500618 int result = -EBUSY;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500619
Matthew Wilcoxeeee3222011-02-14 15:55:33 -0500620 spin_lock_irq(&nvmeq->q_lock);
621 if (bio_list_empty(&nvmeq->sq_cong))
622 result = nvme_submit_bio_queue(nvmeq, ns, bio);
623 if (unlikely(result)) {
624 if (bio_list_empty(&nvmeq->sq_cong))
625 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500626 bio_list_add(&nvmeq->sq_cong, bio);
627 }
Matthew Wilcoxeeee3222011-02-14 15:55:33 -0500628
629 spin_unlock_irq(&nvmeq->q_lock);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500630 put_nvmeq(nvmeq);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500631}
632
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500633static irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq)
634{
Matthew Wilcox82123462011-01-20 13:24:06 -0500635 u16 head, phase;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500636
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500637 head = nvmeq->cq_head;
Matthew Wilcox82123462011-01-20 13:24:06 -0500638 phase = nvmeq->cq_phase;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500639
640 for (;;) {
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400641 void *ctx;
642 nvme_completion_fn fn;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500643 struct nvme_completion cqe = nvmeq->cqes[head];
Matthew Wilcox82123462011-01-20 13:24:06 -0500644 if ((le16_to_cpu(cqe.status) & 1) != phase)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500645 break;
646 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
647 if (++head == nvmeq->q_depth) {
648 head = 0;
Matthew Wilcox82123462011-01-20 13:24:06 -0500649 phase = !phase;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500650 }
651
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400652 ctx = free_cmdid(nvmeq, cqe.command_id, &fn);
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500653 fn(nvmeq->dev, ctx, &cqe);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500654 }
655
656 /* If the controller ignores the cq head doorbell and continuously
657 * writes to the queue, it is theoretically possible to wrap around
658 * the queue twice and mistakenly return IRQ_NONE. Linux only
659 * requires that 0.1% of your interrupts are handled, so this isn't
660 * a big problem.
661 */
Matthew Wilcox82123462011-01-20 13:24:06 -0500662 if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500663 return IRQ_NONE;
664
Matthew Wilcoxf1938f62011-10-20 17:00:41 -0400665 writel(head, nvmeq->q_db + (1 << nvmeq->dev->db_stride));
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500666 nvmeq->cq_head = head;
Matthew Wilcox82123462011-01-20 13:24:06 -0500667 nvmeq->cq_phase = phase;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500668
669 return IRQ_HANDLED;
670}
671
672static irqreturn_t nvme_irq(int irq, void *data)
673{
Matthew Wilcox58ffacb2011-02-06 07:28:06 -0500674 irqreturn_t result;
675 struct nvme_queue *nvmeq = data;
676 spin_lock(&nvmeq->q_lock);
677 result = nvme_process_cq(nvmeq);
678 spin_unlock(&nvmeq->q_lock);
679 return result;
680}
681
682static irqreturn_t nvme_irq_check(int irq, void *data)
683{
684 struct nvme_queue *nvmeq = data;
685 struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
686 if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
687 return IRQ_NONE;
688 return IRQ_WAKE_THREAD;
689}
690
Matthew Wilcox3c0cf132011-02-04 16:03:56 -0500691static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid)
692{
693 spin_lock_irq(&nvmeq->q_lock);
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400694 cancel_cmdid(nvmeq, cmdid, NULL);
Matthew Wilcox3c0cf132011-02-04 16:03:56 -0500695 spin_unlock_irq(&nvmeq->q_lock);
696}
697
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400698struct sync_cmd_info {
699 struct task_struct *task;
700 u32 result;
701 int status;
702};
703
Matthew Wilcox5c1281a2011-12-20 11:54:53 -0500704static void sync_completion(struct nvme_dev *dev, void *ctx,
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400705 struct nvme_completion *cqe)
706{
707 struct sync_cmd_info *cmdinfo = ctx;
708 cmdinfo->result = le32_to_cpup(&cqe->result);
709 cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
710 wake_up_process(cmdinfo->task);
711}
712
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500713/*
714 * Returns 0 on success. If the result is negative, it's a Linux error code;
715 * if the result is positive, it's an NVM Express status code
716 */
Vishal Verma5d0f6132013-03-04 18:40:58 -0700717int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,
718 u32 *result, unsigned timeout)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500719{
720 int cmdid;
721 struct sync_cmd_info cmdinfo;
722
723 cmdinfo.task = current;
724 cmdinfo.status = -EINTR;
725
Matthew Wilcoxc2f5b652011-10-15 07:33:46 -0400726 cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion,
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500727 timeout);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500728 if (cmdid < 0)
729 return cmdid;
730 cmd->common.command_id = cmdid;
731
Matthew Wilcox3c0cf132011-02-04 16:03:56 -0500732 set_current_state(TASK_KILLABLE);
733 nvme_submit_cmd(nvmeq, cmd);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500734 schedule();
735
Matthew Wilcox3c0cf132011-02-04 16:03:56 -0500736 if (cmdinfo.status == -EINTR) {
737 nvme_abort_command(nvmeq, cmdid);
738 return -EINTR;
739 }
740
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500741 if (result)
742 *result = cmdinfo.result;
743
744 return cmdinfo.status;
745}
746
Vishal Verma5d0f6132013-03-04 18:40:58 -0700747int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500748 u32 *result)
749{
Matthew Wilcoxe85248e2011-02-06 18:30:16 -0500750 return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500751}
752
753static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
754{
755 int status;
756 struct nvme_command c;
757
758 memset(&c, 0, sizeof(c));
759 c.delete_queue.opcode = opcode;
760 c.delete_queue.qid = cpu_to_le16(id);
761
762 status = nvme_submit_admin_cmd(dev, &c, NULL);
763 if (status)
764 return -EIO;
765 return 0;
766}
767
768static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
769 struct nvme_queue *nvmeq)
770{
771 int status;
772 struct nvme_command c;
773 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
774
775 memset(&c, 0, sizeof(c));
776 c.create_cq.opcode = nvme_admin_create_cq;
777 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
778 c.create_cq.cqid = cpu_to_le16(qid);
779 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
780 c.create_cq.cq_flags = cpu_to_le16(flags);
781 c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
782
783 status = nvme_submit_admin_cmd(dev, &c, NULL);
784 if (status)
785 return -EIO;
786 return 0;
787}
788
789static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
790 struct nvme_queue *nvmeq)
791{
792 int status;
793 struct nvme_command c;
794 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
795
796 memset(&c, 0, sizeof(c));
797 c.create_sq.opcode = nvme_admin_create_sq;
798 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
799 c.create_sq.sqid = cpu_to_le16(qid);
800 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
801 c.create_sq.sq_flags = cpu_to_le16(flags);
802 c.create_sq.cqid = cpu_to_le16(qid);
803
804 status = nvme_submit_admin_cmd(dev, &c, NULL);
805 if (status)
806 return -EIO;
807 return 0;
808}
809
810static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
811{
812 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
813}
814
815static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
816{
817 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
818}
819
Vishal Verma5d0f6132013-03-04 18:40:58 -0700820int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -0400821 dma_addr_t dma_addr)
822{
823 struct nvme_command c;
824
825 memset(&c, 0, sizeof(c));
826 c.identify.opcode = nvme_admin_identify;
827 c.identify.nsid = cpu_to_le32(nsid);
828 c.identify.prp1 = cpu_to_le64(dma_addr);
829 c.identify.cns = cpu_to_le32(cns);
830
831 return nvme_submit_admin_cmd(dev, &c, NULL);
832}
833
Vishal Verma5d0f6132013-03-04 18:40:58 -0700834int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
Keith Busch08df1e02012-09-21 10:52:13 -0600835 dma_addr_t dma_addr, u32 *result)
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -0400836{
837 struct nvme_command c;
838
839 memset(&c, 0, sizeof(c));
840 c.features.opcode = nvme_admin_get_features;
Keith Buscha42cecc2012-07-25 16:06:38 -0600841 c.features.nsid = cpu_to_le32(nsid);
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -0400842 c.features.prp1 = cpu_to_le64(dma_addr);
843 c.features.fid = cpu_to_le32(fid);
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -0400844
Keith Busch08df1e02012-09-21 10:52:13 -0600845 return nvme_submit_admin_cmd(dev, &c, result);
Matthew Wilcoxdf348132012-01-11 07:29:56 -0700846}
847
Vishal Verma5d0f6132013-03-04 18:40:58 -0700848int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
849 dma_addr_t dma_addr, u32 *result)
Matthew Wilcoxdf348132012-01-11 07:29:56 -0700850{
851 struct nvme_command c;
852
853 memset(&c, 0, sizeof(c));
854 c.features.opcode = nvme_admin_set_features;
855 c.features.prp1 = cpu_to_le64(dma_addr);
856 c.features.fid = cpu_to_le32(fid);
857 c.features.dword11 = cpu_to_le32(dword11);
858
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -0400859 return nvme_submit_admin_cmd(dev, &c, result);
860}
861
Matthew Wilcoxa09115b2012-08-07 15:56:23 -0400862/**
863 * nvme_cancel_ios - Cancel outstanding I/Os
864 * @queue: The queue to cancel I/Os on
865 * @timeout: True to only cancel I/Os which have timed out
866 */
867static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
868{
869 int depth = nvmeq->q_depth - 1;
870 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
871 unsigned long now = jiffies;
872 int cmdid;
873
874 for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
875 void *ctx;
876 nvme_completion_fn fn;
877 static struct nvme_completion cqe = {
Matthew Wilcoxaf2d9ca2013-04-16 15:18:30 -0400878 .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1),
Matthew Wilcoxa09115b2012-08-07 15:56:23 -0400879 };
880
881 if (timeout && !time_after(now, info[cmdid].timeout))
882 continue;
883 dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
884 ctx = cancel_cmdid(nvmeq, cmdid, &fn);
885 fn(nvmeq->dev, ctx, &cqe);
886 }
887}
888
Matthew Wilcox9e866772012-08-03 13:55:56 -0400889static void nvme_free_queue_mem(struct nvme_queue *nvmeq)
890{
891 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
892 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
893 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
894 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
895 kfree(nvmeq);
896}
897
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500898static void nvme_free_queue(struct nvme_dev *dev, int qid)
899{
900 struct nvme_queue *nvmeq = dev->queues[qid];
Matthew Wilcoxaba20802011-03-27 08:52:06 -0400901 int vector = dev->entry[nvmeq->cq_vector].vector;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500902
Matthew Wilcoxa09115b2012-08-07 15:56:23 -0400903 spin_lock_irq(&nvmeq->q_lock);
904 nvme_cancel_ios(nvmeq, false);
Keith Busch32958742012-08-20 14:57:49 -0600905 while (bio_list_peek(&nvmeq->sq_cong)) {
906 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
907 bio_endio(bio, -EIO);
908 }
Matthew Wilcoxa09115b2012-08-07 15:56:23 -0400909 spin_unlock_irq(&nvmeq->q_lock);
910
Matthew Wilcoxaba20802011-03-27 08:52:06 -0400911 irq_set_affinity_hint(vector, NULL);
912 free_irq(vector, nvmeq);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500913
914 /* Don't tell the adapter to delete the admin queue */
915 if (qid) {
916 adapter_delete_sq(dev, qid);
917 adapter_delete_cq(dev, qid);
918 }
919
Matthew Wilcox9e866772012-08-03 13:55:56 -0400920 nvme_free_queue_mem(nvmeq);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500921}
922
923static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
924 int depth, int vector)
925{
926 struct device *dmadev = &dev->pci_dev->dev;
Keith Buscha0cadb82012-07-27 13:57:23 -0400927 unsigned extra = DIV_ROUND_UP(depth, 8) + (depth *
928 sizeof(struct nvme_cmd_info));
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500929 struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
930 if (!nvmeq)
931 return NULL;
932
933 nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth),
934 &nvmeq->cq_dma_addr, GFP_KERNEL);
935 if (!nvmeq->cqes)
936 goto free_nvmeq;
937 memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth));
938
939 nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
940 &nvmeq->sq_dma_addr, GFP_KERNEL);
941 if (!nvmeq->sq_cmds)
942 goto free_cqdma;
943
944 nvmeq->q_dmadev = dmadev;
Matthew Wilcox091b6092011-02-10 09:56:01 -0500945 nvmeq->dev = dev;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500946 spin_lock_init(&nvmeq->q_lock);
947 nvmeq->cq_head = 0;
Matthew Wilcox82123462011-01-20 13:24:06 -0500948 nvmeq->cq_phase = 1;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500949 init_waitqueue_head(&nvmeq->sq_full);
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -0500950 init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500951 bio_list_init(&nvmeq->sq_cong);
Matthew Wilcoxf1938f62011-10-20 17:00:41 -0400952 nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)];
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500953 nvmeq->q_depth = depth;
954 nvmeq->cq_vector = vector;
955
956 return nvmeq;
957
958 free_cqdma:
959 dma_free_coherent(dmadev, CQ_SIZE(nvmeq->q_depth), (void *)nvmeq->cqes,
960 nvmeq->cq_dma_addr);
961 free_nvmeq:
962 kfree(nvmeq);
963 return NULL;
964}
965
Matthew Wilcox30010822011-01-20 09:10:15 -0500966static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
967 const char *name)
968{
Matthew Wilcox58ffacb2011-02-06 07:28:06 -0500969 if (use_threaded_interrupts)
970 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
Matthew Wilcoxec6ce612011-02-06 09:01:00 -0500971 nvme_irq_check, nvme_irq,
Matthew Wilcox58ffacb2011-02-06 07:28:06 -0500972 IRQF_DISABLED | IRQF_SHARED,
973 name, nvmeq);
Matthew Wilcox30010822011-01-20 09:10:15 -0500974 return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
975 IRQF_DISABLED | IRQF_SHARED, name, nvmeq);
976}
977
Greg Kroah-Hartman8d85fce2012-12-21 15:13:49 -0800978static struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, int qid,
979 int cq_size, int vector)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500980{
981 int result;
982 struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector);
983
Matthew Wilcox3f85d502011-02-01 08:39:04 -0500984 if (!nvmeq)
Matthew Wilcox6f0f5442011-05-11 13:30:59 -0700985 return ERR_PTR(-ENOMEM);
Matthew Wilcox3f85d502011-02-01 08:39:04 -0500986
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500987 result = adapter_alloc_cq(dev, qid, nvmeq);
988 if (result < 0)
989 goto free_nvmeq;
990
991 result = adapter_alloc_sq(dev, qid, nvmeq);
992 if (result < 0)
993 goto release_cq;
994
Matthew Wilcox30010822011-01-20 09:10:15 -0500995 result = queue_request_irq(dev, nvmeq, "nvme");
Matthew Wilcoxb60503b2011-01-20 12:50:14 -0500996 if (result < 0)
997 goto release_sq;
998
999 return nvmeq;
1000
1001 release_sq:
1002 adapter_delete_sq(dev, qid);
1003 release_cq:
1004 adapter_delete_cq(dev, qid);
1005 free_nvmeq:
1006 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1007 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1008 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1009 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1010 kfree(nvmeq);
Matthew Wilcox6f0f5442011-05-11 13:30:59 -07001011 return ERR_PTR(result);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001012}
1013
Greg Kroah-Hartman8d85fce2012-12-21 15:13:49 -08001014static int nvme_configure_admin_queue(struct nvme_dev *dev)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001015{
Matthew Wilcox9e866772012-08-03 13:55:56 -04001016 int result = 0;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001017 u32 aqa;
Matthew Wilcox22605f92011-04-19 15:04:20 -04001018 u64 cap;
1019 unsigned long timeout;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001020 struct nvme_queue *nvmeq;
1021
1022 dev->dbs = ((void __iomem *)dev->bar) + 4096;
1023
1024 nvmeq = nvme_alloc_queue(dev, 0, 64, 0);
Matthew Wilcox3f85d502011-02-01 08:39:04 -05001025 if (!nvmeq)
1026 return -ENOMEM;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001027
1028 aqa = nvmeq->q_depth - 1;
1029 aqa |= aqa << 16;
1030
1031 dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM;
1032 dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
1033 dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
Matthew Wilcox7f53f9d2011-03-22 15:55:45 -04001034 dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001035
Shane Michael Matthews5911f202011-02-01 11:31:55 -05001036 writel(0, &dev->bar->cc);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001037 writel(aqa, &dev->bar->aqa);
1038 writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1039 writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
1040 writel(dev->ctrl_config, &dev->bar->cc);
1041
Matthew Wilcox22605f92011-04-19 15:04:20 -04001042 cap = readq(&dev->bar->cap);
1043 timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
Matthew Wilcoxf1938f62011-10-20 17:00:41 -04001044 dev->db_stride = NVME_CAP_STRIDE(cap);
Matthew Wilcox22605f92011-04-19 15:04:20 -04001045
Matthew Wilcox9e866772012-08-03 13:55:56 -04001046 while (!result && !(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001047 msleep(100);
1048 if (fatal_signal_pending(current))
Matthew Wilcox9e866772012-08-03 13:55:56 -04001049 result = -EINTR;
Matthew Wilcox22605f92011-04-19 15:04:20 -04001050 if (time_after(jiffies, timeout)) {
1051 dev_err(&dev->pci_dev->dev,
1052 "Device not ready; aborting initialisation\n");
Matthew Wilcox9e866772012-08-03 13:55:56 -04001053 result = -ENODEV;
Matthew Wilcox22605f92011-04-19 15:04:20 -04001054 }
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001055 }
1056
Keith Busch025c5572013-05-01 13:07:51 -06001057 if (result)
1058 goto free_q;
Matthew Wilcox9e866772012-08-03 13:55:56 -04001059
Matthew Wilcox30010822011-01-20 09:10:15 -05001060 result = queue_request_irq(dev, nvmeq, "nvme admin");
Keith Busch025c5572013-05-01 13:07:51 -06001061 if (result)
1062 goto free_q;
1063
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001064 dev->queues[0] = nvmeq;
1065 return result;
Keith Busch025c5572013-05-01 13:07:51 -06001066
1067 free_q:
1068 nvme_free_queue_mem(nvmeq);
1069 return result;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001070}
1071
Vishal Verma5d0f6132013-03-04 18:40:58 -07001072struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001073 unsigned long addr, unsigned length)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001074{
Matthew Wilcox36c14ed2011-01-24 07:52:07 -05001075 int i, err, count, nents, offset;
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001076 struct scatterlist *sg;
1077 struct page **pages;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001078 struct nvme_iod *iod;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001079
Matthew Wilcox36c14ed2011-01-24 07:52:07 -05001080 if (addr & 3)
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001081 return ERR_PTR(-EINVAL);
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001082 if (!length)
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001083 return ERR_PTR(-EINVAL);
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001084
Matthew Wilcox36c14ed2011-01-24 07:52:07 -05001085 offset = offset_in_page(addr);
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001086 count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
1087 pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
Dan Carpenter22fff822012-01-20 07:55:30 -05001088 if (!pages)
1089 return ERR_PTR(-ENOMEM);
Matthew Wilcox36c14ed2011-01-24 07:52:07 -05001090
1091 err = get_user_pages_fast(addr, count, 1, pages);
1092 if (err < count) {
1093 count = err;
1094 err = -EFAULT;
1095 goto put_pages;
1096 }
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001097
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001098 iod = nvme_alloc_iod(count, length, GFP_KERNEL);
1099 sg = iod->sg;
Matthew Wilcox36c14ed2011-01-24 07:52:07 -05001100 sg_init_table(sg, count);
Matthew Wilcoxd0ba1e42011-09-13 17:01:39 -04001101 for (i = 0; i < count; i++) {
1102 sg_set_page(&sg[i], pages[i],
1103 min_t(int, length, PAGE_SIZE - offset), offset);
1104 length -= (PAGE_SIZE - offset);
1105 offset = 0;
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001106 }
Matthew Wilcoxfe304c42012-01-06 13:49:25 -07001107 sg_mark_end(&sg[i - 1]);
Matthew Wilcox1c2ad9f2012-01-06 13:52:56 -07001108 iod->nents = count;
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001109
1110 err = -ENOMEM;
1111 nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
1112 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
Matthew Wilcox36c14ed2011-01-24 07:52:07 -05001113 if (!nents)
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001114 goto free_iod;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001115
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001116 kfree(pages);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001117 return iod;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001118
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001119 free_iod:
1120 kfree(iod);
Matthew Wilcox36c14ed2011-01-24 07:52:07 -05001121 put_pages:
1122 for (i = 0; i < count; i++)
1123 put_page(pages[i]);
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001124 kfree(pages);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001125 return ERR_PTR(err);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001126}
1127
Vishal Verma5d0f6132013-03-04 18:40:58 -07001128void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
Matthew Wilcox1c2ad9f2012-01-06 13:52:56 -07001129 struct nvme_iod *iod)
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001130{
Matthew Wilcox1c2ad9f2012-01-06 13:52:56 -07001131 int i;
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001132
Matthew Wilcox1c2ad9f2012-01-06 13:52:56 -07001133 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
1134 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001135
Matthew Wilcox1c2ad9f2012-01-06 13:52:56 -07001136 for (i = 0; i < iod->nents; i++)
1137 put_page(sg_page(&iod->sg[i]));
Matthew Wilcox7fc3cda2011-01-26 17:05:50 -05001138}
1139
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001140static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1141{
1142 struct nvme_dev *dev = ns->dev;
1143 struct nvme_queue *nvmeq;
1144 struct nvme_user_io io;
1145 struct nvme_command c;
1146 unsigned length;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001147 int status;
1148 struct nvme_iod *iod;
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001149
1150 if (copy_from_user(&io, uio, sizeof(io)))
1151 return -EFAULT;
Matthew Wilcox6c7d4942011-03-21 09:48:57 -04001152 length = (io.nblocks + 1) << ns->lba_shift;
1153
1154 switch (io.opcode) {
1155 case nvme_cmd_write:
1156 case nvme_cmd_read:
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001157 case nvme_cmd_compare:
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001158 iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
Matthew Wilcox64132142011-08-09 12:56:37 -04001159 break;
Matthew Wilcox6c7d4942011-03-21 09:48:57 -04001160 default:
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001161 return -EINVAL;
Matthew Wilcox6c7d4942011-03-21 09:48:57 -04001162 }
1163
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001164 if (IS_ERR(iod))
1165 return PTR_ERR(iod);
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001166
1167 memset(&c, 0, sizeof(c));
1168 c.rw.opcode = io.opcode;
1169 c.rw.flags = io.flags;
Matthew Wilcox6c7d4942011-03-21 09:48:57 -04001170 c.rw.nsid = cpu_to_le32(ns->ns_id);
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001171 c.rw.slba = cpu_to_le64(io.slba);
Matthew Wilcox6c7d4942011-03-21 09:48:57 -04001172 c.rw.length = cpu_to_le16(io.nblocks);
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001173 c.rw.control = cpu_to_le16(io.control);
Matthew Wilcox1c9b5262013-04-16 15:21:06 -04001174 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1175 c.rw.reftag = cpu_to_le32(io.reftag);
1176 c.rw.apptag = cpu_to_le16(io.apptag);
1177 c.rw.appmask = cpu_to_le16(io.appmask);
Shane Michael Matthewse025344c2011-02-10 08:51:24 -05001178 /* XXX: metadata */
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001179 length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL);
Shane Michael Matthewse025344c2011-02-10 08:51:24 -05001180
Matthew Wilcox040a93b2011-12-20 11:04:12 -05001181 nvmeq = get_nvmeq(dev);
Matthew Wilcoxfa922822011-03-16 16:29:00 -04001182 /*
1183 * Since nvme_submit_sync_cmd sleeps, we can't keep preemption
Matthew Wilcoxb1ad37e2011-02-04 16:14:30 -05001184 * disabled. We may be preempted at any point, and be rescheduled
1185 * to a different CPU. That will cause cacheline bouncing, but no
1186 * additional races since q_lock already protects against other CPUs.
1187 */
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001188 put_nvmeq(nvmeq);
Matthew Wilcoxb77954c2011-05-12 13:51:41 -04001189 if (length != (io.nblocks + 1) << ns->lba_shift)
1190 status = -ENOMEM;
1191 else
Matthew Wilcoxff976d72011-12-20 13:53:01 -05001192 status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT);
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001193
Matthew Wilcox1c2ad9f2012-01-06 13:52:56 -07001194 nvme_unmap_user_pages(dev, io.opcode & 1, iod);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001195 nvme_free_iod(dev, iod);
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001196 return status;
1197}
1198
Keith Busch50af8ba2012-07-25 16:07:55 -06001199static int nvme_user_admin_cmd(struct nvme_dev *dev,
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001200 struct nvme_admin_cmd __user *ucmd)
Matthew Wilcox6ee44cd2011-02-03 10:58:26 -05001201{
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001202 struct nvme_admin_cmd cmd;
Matthew Wilcox6ee44cd2011-02-03 10:58:26 -05001203 struct nvme_command c;
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001204 int status, length;
Keith Buschc7d36ab2012-07-27 11:53:28 -06001205 struct nvme_iod *uninitialized_var(iod);
Matthew Wilcox6ee44cd2011-02-03 10:58:26 -05001206
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001207 if (!capable(CAP_SYS_ADMIN))
1208 return -EACCES;
1209 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
Matthew Wilcox6ee44cd2011-02-03 10:58:26 -05001210 return -EFAULT;
Matthew Wilcox6ee44cd2011-02-03 10:58:26 -05001211
1212 memset(&c, 0, sizeof(c));
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001213 c.common.opcode = cmd.opcode;
1214 c.common.flags = cmd.flags;
1215 c.common.nsid = cpu_to_le32(cmd.nsid);
1216 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1217 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1218 c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1219 c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1220 c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1221 c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1222 c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1223 c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1224
1225 length = cmd.data_len;
1226 if (cmd.data_len) {
Matthew Wilcox49742182012-01-06 13:42:45 -07001227 iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
1228 length);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001229 if (IS_ERR(iod))
1230 return PTR_ERR(iod);
1231 length = nvme_setup_prps(dev, &c.common, iod, length,
1232 GFP_KERNEL);
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001233 }
1234
1235 if (length != cmd.data_len)
Matthew Wilcoxb77954c2011-05-12 13:51:41 -04001236 status = -ENOMEM;
1237 else
Keith Buschf4f117f2012-09-21 10:49:05 -06001238 status = nvme_submit_admin_cmd(dev, &c, &cmd.result);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001239
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001240 if (cmd.data_len) {
Matthew Wilcox1c2ad9f2012-01-06 13:52:56 -07001241 nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
Matthew Wilcoxeca18b22011-12-20 13:34:52 -05001242 nvme_free_iod(dev, iod);
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001243 }
Keith Buschf4f117f2012-09-21 10:49:05 -06001244
1245 if (!status && copy_to_user(&ucmd->result, &cmd.result,
1246 sizeof(cmd.result)))
1247 status = -EFAULT;
1248
Matthew Wilcox6ee44cd2011-02-03 10:58:26 -05001249 return status;
1250}
1251
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001252static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1253 unsigned long arg)
1254{
1255 struct nvme_ns *ns = bdev->bd_disk->private_data;
1256
1257 switch (cmd) {
Matthew Wilcox6bbf1ac2011-05-20 13:03:42 -04001258 case NVME_IOCTL_ID:
1259 return ns->ns_id;
1260 case NVME_IOCTL_ADMIN_CMD:
Keith Busch50af8ba2012-07-25 16:07:55 -06001261 return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
Matthew Wilcoxa53295b2011-02-01 16:13:29 -05001262 case NVME_IOCTL_SUBMIT_IO:
1263 return nvme_submit_io(ns, (void __user *)arg);
Vishal Verma5d0f6132013-03-04 18:40:58 -07001264 case SG_GET_VERSION_NUM:
1265 return nvme_sg_get_version_num((void __user *)arg);
1266 case SG_IO:
1267 return nvme_sg_io(ns, (void __user *)arg);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001268 default:
1269 return -ENOTTY;
1270 }
1271}
1272
1273static const struct block_device_operations nvme_fops = {
1274 .owner = THIS_MODULE,
1275 .ioctl = nvme_ioctl,
Matthew Wilcox49481682011-03-19 14:55:38 -04001276 .compat_ioctl = nvme_ioctl,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001277};
1278
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001279static void nvme_resubmit_bios(struct nvme_queue *nvmeq)
1280{
1281 while (bio_list_peek(&nvmeq->sq_cong)) {
1282 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1283 struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data;
1284 if (nvme_submit_bio_queue(nvmeq, ns, bio)) {
1285 bio_list_add_head(&nvmeq->sq_cong, bio);
1286 break;
1287 }
Matthew Wilcox3cb967c2011-03-16 16:45:49 -04001288 if (bio_list_empty(&nvmeq->sq_cong))
1289 remove_wait_queue(&nvmeq->sq_full,
1290 &nvmeq->sq_cong_wait);
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001291 }
1292}
1293
1294static int nvme_kthread(void *data)
1295{
1296 struct nvme_dev *dev;
1297
1298 while (!kthread_should_stop()) {
Arjan van de Ven564a2322013-05-01 16:38:23 -04001299 set_current_state(TASK_INTERRUPTIBLE);
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001300 spin_lock(&dev_list_lock);
1301 list_for_each_entry(dev, &dev_list, node) {
1302 int i;
1303 for (i = 0; i < dev->queue_count; i++) {
1304 struct nvme_queue *nvmeq = dev->queues[i];
Matthew Wilcox740216f2011-02-15 16:28:20 -05001305 if (!nvmeq)
1306 continue;
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001307 spin_lock_irq(&nvmeq->q_lock);
1308 if (nvme_process_cq(nvmeq))
1309 printk("process_cq did something\n");
Matthew Wilcoxa09115b2012-08-07 15:56:23 -04001310 nvme_cancel_ios(nvmeq, true);
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001311 nvme_resubmit_bios(nvmeq);
1312 spin_unlock_irq(&nvmeq->q_lock);
1313 }
1314 }
1315 spin_unlock(&dev_list_lock);
Arjan van de Venacb7aa02013-02-04 14:44:33 -08001316 schedule_timeout(round_jiffies_relative(HZ));
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001317 }
1318 return 0;
1319}
1320
Matthew Wilcox5aff9382011-05-06 08:45:47 -04001321static DEFINE_IDA(nvme_index_ida);
1322
1323static int nvme_get_ns_idx(void)
1324{
1325 int index, error;
1326
1327 do {
1328 if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL))
1329 return -1;
1330
1331 spin_lock(&dev_list_lock);
1332 error = ida_get_new(&nvme_index_ida, &index);
1333 spin_unlock(&dev_list_lock);
1334 } while (error == -EAGAIN);
1335
1336 if (error)
1337 index = -1;
1338 return index;
1339}
1340
1341static void nvme_put_ns_idx(int index)
1342{
1343 spin_lock(&dev_list_lock);
1344 ida_remove(&nvme_index_ida, index);
1345 spin_unlock(&dev_list_lock);
1346}
1347
Keith Busch0e5e4f02012-11-09 16:33:05 -07001348static void nvme_config_discard(struct nvme_ns *ns)
1349{
1350 u32 logical_block_size = queue_logical_block_size(ns->queue);
1351 ns->queue->limits.discard_zeroes_data = 0;
1352 ns->queue->limits.discard_alignment = logical_block_size;
1353 ns->queue->limits.discard_granularity = logical_block_size;
1354 ns->queue->limits.max_discard_sectors = 0xffffffff;
1355 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1356}
1357
Matthew Wilcox5aff9382011-05-06 08:45:47 -04001358static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001359 struct nvme_id_ns *id, struct nvme_lba_range_type *rt)
1360{
1361 struct nvme_ns *ns;
1362 struct gendisk *disk;
1363 int lbaf;
1364
1365 if (rt->attributes & NVME_LBART_ATTRIB_HIDE)
1366 return NULL;
1367
1368 ns = kzalloc(sizeof(*ns), GFP_KERNEL);
1369 if (!ns)
1370 return NULL;
1371 ns->queue = blk_alloc_queue(GFP_KERNEL);
1372 if (!ns->queue)
1373 goto out_free_ns;
Matthew Wilcox4eeb9212012-01-10 14:35:08 -07001374 ns->queue->queue_flags = QUEUE_FLAG_DEFAULT;
1375 queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
1376 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001377 blk_queue_make_request(ns->queue, nvme_make_request);
1378 ns->dev = dev;
1379 ns->queue->queuedata = ns;
1380
1381 disk = alloc_disk(NVME_MINORS);
1382 if (!disk)
1383 goto out_free_queue;
Matthew Wilcox5aff9382011-05-06 08:45:47 -04001384 ns->ns_id = nsid;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001385 ns->disk = disk;
1386 lbaf = id->flbas & 0xf;
1387 ns->lba_shift = id->lbaf[lbaf].ds;
Keith Busche9ef4632012-07-24 15:01:04 -06001388 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
Keith Busch8fc23e02012-07-26 11:29:57 -06001389 if (dev->max_hw_sectors)
1390 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001391
1392 disk->major = nvme_major;
1393 disk->minors = NVME_MINORS;
Matthew Wilcox5aff9382011-05-06 08:45:47 -04001394 disk->first_minor = NVME_MINORS * nvme_get_ns_idx();
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001395 disk->fops = &nvme_fops;
1396 disk->private_data = ns;
1397 disk->queue = ns->queue;
Matthew Wilcox388f0372011-02-01 12:49:38 -05001398 disk->driverfs_dev = &dev->pci_dev->dev;
Matthew Wilcox5aff9382011-05-06 08:45:47 -04001399 sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001400 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1401
Keith Busch0e5e4f02012-11-09 16:33:05 -07001402 if (dev->oncs & NVME_CTRL_ONCS_DSM)
1403 nvme_config_discard(ns);
1404
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001405 return ns;
1406
1407 out_free_queue:
1408 blk_cleanup_queue(ns->queue);
1409 out_free_ns:
1410 kfree(ns);
1411 return NULL;
1412}
1413
1414static void nvme_ns_free(struct nvme_ns *ns)
1415{
Matthew Wilcox5aff9382011-05-06 08:45:47 -04001416 int index = ns->disk->first_minor / NVME_MINORS;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001417 put_disk(ns->disk);
Matthew Wilcox5aff9382011-05-06 08:45:47 -04001418 nvme_put_ns_idx(index);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001419 blk_cleanup_queue(ns->queue);
1420 kfree(ns);
1421}
1422
Matthew Wilcoxb3b06812011-01-20 09:14:34 -05001423static int set_queue_count(struct nvme_dev *dev, int count)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001424{
1425 int status;
1426 u32 result;
Matthew Wilcoxb3b06812011-01-20 09:14:34 -05001427 u32 q_count = (count - 1) | ((count - 1) << 16);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001428
Matthew Wilcoxdf348132012-01-11 07:29:56 -07001429 status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001430 &result);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001431 if (status)
1432 return -EIO;
1433 return min(result & 0xffff, result >> 16) + 1;
1434}
1435
Greg Kroah-Hartman8d85fce2012-12-21 15:13:49 -08001436static int nvme_setup_io_queues(struct nvme_dev *dev)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001437{
Keith Buscha0cadb82012-07-27 13:57:23 -04001438 int result, cpu, i, nr_io_queues, db_bar_size, q_depth;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001439
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001440 nr_io_queues = num_online_cpus();
1441 result = set_queue_count(dev, nr_io_queues);
Matthew Wilcox1b234842011-01-20 13:01:49 -05001442 if (result < 0)
1443 return result;
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001444 if (result < nr_io_queues)
1445 nr_io_queues = result;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001446
Matthew Wilcox1b234842011-01-20 13:01:49 -05001447 /* Deregister the admin queue's interrupt */
1448 free_irq(dev->entry[0].vector, dev->queues[0]);
1449
Matthew Wilcoxf1938f62011-10-20 17:00:41 -04001450 db_bar_size = 4096 + ((nr_io_queues + 1) << (dev->db_stride + 3));
1451 if (db_bar_size > 8192) {
1452 iounmap(dev->bar);
1453 dev->bar = ioremap(pci_resource_start(dev->pci_dev, 0),
1454 db_bar_size);
1455 dev->dbs = ((void __iomem *)dev->bar) + 4096;
1456 dev->queues[0]->q_db = dev->dbs;
1457 }
1458
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001459 for (i = 0; i < nr_io_queues; i++)
Matthew Wilcox1b234842011-01-20 13:01:49 -05001460 dev->entry[i].entry = i;
1461 for (;;) {
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001462 result = pci_enable_msix(dev->pci_dev, dev->entry,
1463 nr_io_queues);
Matthew Wilcox1b234842011-01-20 13:01:49 -05001464 if (result == 0) {
1465 break;
1466 } else if (result > 0) {
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001467 nr_io_queues = result;
Matthew Wilcox1b234842011-01-20 13:01:49 -05001468 continue;
1469 } else {
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001470 nr_io_queues = 1;
Matthew Wilcox1b234842011-01-20 13:01:49 -05001471 break;
1472 }
1473 }
1474
1475 result = queue_request_irq(dev, dev->queues[0], "nvme admin");
1476 /* XXX: handle failure here */
1477
1478 cpu = cpumask_first(cpu_online_mask);
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001479 for (i = 0; i < nr_io_queues; i++) {
Matthew Wilcox1b234842011-01-20 13:01:49 -05001480 irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu));
1481 cpu = cpumask_next(cpu, cpu_online_mask);
1482 }
1483
Keith Buscha0cadb82012-07-27 13:57:23 -04001484 q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
1485 NVME_Q_DEPTH);
Matthew Wilcoxb348b7d2011-02-15 16:16:02 -05001486 for (i = 0; i < nr_io_queues; i++) {
Keith Buscha0cadb82012-07-27 13:57:23 -04001487 dev->queues[i + 1] = nvme_create_queue(dev, i + 1, q_depth, i);
Matthew Wilcox6f0f5442011-05-11 13:30:59 -07001488 if (IS_ERR(dev->queues[i + 1]))
1489 return PTR_ERR(dev->queues[i + 1]);
Matthew Wilcox1b234842011-01-20 13:01:49 -05001490 dev->queue_count++;
1491 }
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001492
Matthew Wilcox9ecdc942011-03-16 16:52:19 -04001493 for (; i < num_possible_cpus(); i++) {
1494 int target = i % rounddown_pow_of_two(dev->queue_count - 1);
1495 dev->queues[i + 1] = dev->queues[target + 1];
1496 }
1497
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001498 return 0;
1499}
1500
1501static void nvme_free_queues(struct nvme_dev *dev)
1502{
1503 int i;
1504
1505 for (i = dev->queue_count - 1; i >= 0; i--)
1506 nvme_free_queue(dev, i);
1507}
1508
Matthew Wilcox422ef0c2013-04-16 11:22:36 -04001509/*
1510 * Return: error value if an error occurred setting up the queues or calling
1511 * Identify Device. 0 if these succeeded, even if adding some of the
1512 * namespaces failed. At the moment, these failures are silent. TBD which
1513 * failures should be reported.
1514 */
Greg Kroah-Hartman8d85fce2012-12-21 15:13:49 -08001515static int nvme_dev_add(struct nvme_dev *dev)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001516{
1517 int res, nn, i;
1518 struct nvme_ns *ns, *next;
Matthew Wilcox51814232011-02-01 16:18:08 -05001519 struct nvme_id_ctrl *ctrl;
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001520 struct nvme_id_ns *id_ns;
1521 void *mem;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001522 dma_addr_t dma_addr;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001523
1524 res = nvme_setup_io_queues(dev);
1525 if (res)
1526 return res;
1527
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001528 mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001529 GFP_KERNEL);
1530
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001531 res = nvme_identify(dev, 0, 1, dma_addr);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001532 if (res) {
1533 res = -EIO;
1534 goto out_free;
1535 }
1536
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001537 ctrl = mem;
Matthew Wilcox51814232011-02-01 16:18:08 -05001538 nn = le32_to_cpup(&ctrl->nn);
Keith Busch0e5e4f02012-11-09 16:33:05 -07001539 dev->oncs = le16_to_cpup(&ctrl->oncs);
Matthew Wilcox51814232011-02-01 16:18:08 -05001540 memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
1541 memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
1542 memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
Keith Busch8fc23e02012-07-26 11:29:57 -06001543 if (ctrl->mdts) {
1544 int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
1545 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
1546 }
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001547
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001548 id_ns = mem;
Matthew Wilcox2b2c1892011-10-07 13:10:13 -04001549 for (i = 1; i <= nn; i++) {
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001550 res = nvme_identify(dev, i, 0, dma_addr);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001551 if (res)
1552 continue;
1553
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001554 if (id_ns->ncap == 0)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001555 continue;
1556
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001557 res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i,
Keith Busch08df1e02012-09-21 10:52:13 -06001558 dma_addr + 4096, NULL);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001559 if (res)
Keith Busch12209032013-01-31 14:40:38 -07001560 memset(mem + 4096, 0, 4096);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001561
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001562 ns = nvme_alloc_ns(dev, i, mem, mem + 4096);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001563 if (ns)
1564 list_add_tail(&ns->list, &dev->namespaces);
1565 }
1566 list_for_each_entry(ns, &dev->namespaces, list)
1567 add_disk(ns->disk);
Matthew Wilcox422ef0c2013-04-16 11:22:36 -04001568 res = 0;
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001569 goto out;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001570
1571 out_free:
1572 list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
1573 list_del(&ns->list);
1574 nvme_ns_free(ns);
1575 }
1576
Matthew Wilcoxbc5fc7e2011-09-19 17:08:14 -04001577 out:
Matthew Wilcox684f5c22011-09-19 17:14:53 -04001578 dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001579 return res;
1580}
1581
1582static int nvme_dev_remove(struct nvme_dev *dev)
1583{
1584 struct nvme_ns *ns, *next;
1585
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001586 spin_lock(&dev_list_lock);
1587 list_del(&dev->node);
1588 spin_unlock(&dev_list_lock);
1589
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001590 list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
1591 list_del(&ns->list);
1592 del_gendisk(ns->disk);
1593 nvme_ns_free(ns);
1594 }
1595
1596 nvme_free_queues(dev);
1597
1598 return 0;
1599}
1600
Matthew Wilcox091b6092011-02-10 09:56:01 -05001601static int nvme_setup_prp_pools(struct nvme_dev *dev)
1602{
1603 struct device *dmadev = &dev->pci_dev->dev;
1604 dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
1605 PAGE_SIZE, PAGE_SIZE, 0);
1606 if (!dev->prp_page_pool)
1607 return -ENOMEM;
1608
Matthew Wilcox99802a72011-02-10 10:30:34 -05001609 /* Optimisation for I/Os between 4k and 128k */
1610 dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
1611 256, 256, 0);
1612 if (!dev->prp_small_pool) {
1613 dma_pool_destroy(dev->prp_page_pool);
1614 return -ENOMEM;
1615 }
Matthew Wilcox091b6092011-02-10 09:56:01 -05001616 return 0;
1617}
1618
1619static void nvme_release_prp_pools(struct nvme_dev *dev)
1620{
1621 dma_pool_destroy(dev->prp_page_pool);
Matthew Wilcox99802a72011-02-10 10:30:34 -05001622 dma_pool_destroy(dev->prp_small_pool);
Matthew Wilcox091b6092011-02-10 09:56:01 -05001623}
1624
Quoc-Son Anhcd58ad72012-02-21 16:50:53 -07001625static DEFINE_IDA(nvme_instance_ida);
1626
1627static int nvme_set_instance(struct nvme_dev *dev)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001628{
Quoc-Son Anhcd58ad72012-02-21 16:50:53 -07001629 int instance, error;
1630
1631 do {
1632 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
1633 return -ENODEV;
1634
1635 spin_lock(&dev_list_lock);
1636 error = ida_get_new(&nvme_instance_ida, &instance);
1637 spin_unlock(&dev_list_lock);
1638 } while (error == -EAGAIN);
1639
1640 if (error)
1641 return -ENODEV;
1642
1643 dev->instance = instance;
1644 return 0;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001645}
1646
1647static void nvme_release_instance(struct nvme_dev *dev)
1648{
Quoc-Son Anhcd58ad72012-02-21 16:50:53 -07001649 spin_lock(&dev_list_lock);
1650 ida_remove(&nvme_instance_ida, dev->instance);
1651 spin_unlock(&dev_list_lock);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001652}
1653
Keith Busch5e82e952013-02-19 10:17:58 -07001654static void nvme_free_dev(struct kref *kref)
1655{
1656 struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
1657 nvme_dev_remove(dev);
1658 pci_disable_msix(dev->pci_dev);
1659 iounmap(dev->bar);
1660 nvme_release_instance(dev);
1661 nvme_release_prp_pools(dev);
1662 pci_disable_device(dev->pci_dev);
1663 pci_release_regions(dev->pci_dev);
1664 kfree(dev->queues);
1665 kfree(dev->entry);
1666 kfree(dev);
1667}
1668
1669static int nvme_dev_open(struct inode *inode, struct file *f)
1670{
1671 struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev,
1672 miscdev);
1673 kref_get(&dev->kref);
1674 f->private_data = dev;
1675 return 0;
1676}
1677
1678static int nvme_dev_release(struct inode *inode, struct file *f)
1679{
1680 struct nvme_dev *dev = f->private_data;
1681 kref_put(&dev->kref, nvme_free_dev);
1682 return 0;
1683}
1684
1685static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1686{
1687 struct nvme_dev *dev = f->private_data;
1688 switch (cmd) {
1689 case NVME_IOCTL_ADMIN_CMD:
1690 return nvme_user_admin_cmd(dev, (void __user *)arg);
1691 default:
1692 return -ENOTTY;
1693 }
1694}
1695
1696static const struct file_operations nvme_dev_fops = {
1697 .owner = THIS_MODULE,
1698 .open = nvme_dev_open,
1699 .release = nvme_dev_release,
1700 .unlocked_ioctl = nvme_dev_ioctl,
1701 .compat_ioctl = nvme_dev_ioctl,
1702};
1703
Greg Kroah-Hartman8d85fce2012-12-21 15:13:49 -08001704static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001705{
Matthew Wilcox574e8b92011-02-01 16:24:35 -05001706 int bars, result = -ENOMEM;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001707 struct nvme_dev *dev;
1708
1709 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1710 if (!dev)
1711 return -ENOMEM;
1712 dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry),
1713 GFP_KERNEL);
1714 if (!dev->entry)
1715 goto free;
Matthew Wilcox1b234842011-01-20 13:01:49 -05001716 dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *),
1717 GFP_KERNEL);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001718 if (!dev->queues)
1719 goto free;
1720
Shane Michael Matthews0ee5a7d2011-02-01 08:49:30 -05001721 if (pci_enable_device_mem(pdev))
1722 goto free;
Matthew Wilcoxf64d3362011-02-01 09:01:59 -05001723 pci_set_master(pdev);
Matthew Wilcox574e8b92011-02-01 16:24:35 -05001724 bars = pci_select_bars(pdev, IORESOURCE_MEM);
1725 if (pci_request_selected_regions(pdev, bars, "nvme"))
1726 goto disable;
Shane Michael Matthews0ee5a7d2011-02-01 08:49:30 -05001727
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001728 INIT_LIST_HEAD(&dev->namespaces);
1729 dev->pci_dev = pdev;
1730 pci_set_drvdata(pdev, dev);
Matthew Wilcox29303532011-02-01 16:23:39 -05001731 dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1732 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
Quoc-Son Anhcd58ad72012-02-21 16:50:53 -07001733 result = nvme_set_instance(dev);
1734 if (result)
1735 goto disable;
1736
Matthew Wilcox53c95772011-01-20 13:42:34 -05001737 dev->entry[0].vector = pdev->irq;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001738
Matthew Wilcox091b6092011-02-10 09:56:01 -05001739 result = nvme_setup_prp_pools(dev);
1740 if (result)
1741 goto disable_msix;
1742
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001743 dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
1744 if (!dev->bar) {
1745 result = -ENOMEM;
Matthew Wilcox574e8b92011-02-01 16:24:35 -05001746 goto disable_msix;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001747 }
1748
1749 result = nvme_configure_admin_queue(dev);
1750 if (result)
1751 goto unmap;
1752 dev->queue_count++;
1753
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001754 spin_lock(&dev_list_lock);
1755 list_add(&dev->node, &dev_list);
1756 spin_unlock(&dev_list_lock);
1757
Matthew Wilcox740216f2011-02-15 16:28:20 -05001758 result = nvme_dev_add(dev);
1759 if (result)
1760 goto delete;
1761
Keith Busch5e82e952013-02-19 10:17:58 -07001762 scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance);
1763 dev->miscdev.minor = MISC_DYNAMIC_MINOR;
1764 dev->miscdev.parent = &pdev->dev;
1765 dev->miscdev.name = dev->name;
1766 dev->miscdev.fops = &nvme_dev_fops;
1767 result = misc_register(&dev->miscdev);
1768 if (result)
1769 goto remove;
1770
1771 kref_init(&dev->kref);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001772 return 0;
1773
Keith Busch5e82e952013-02-19 10:17:58 -07001774 remove:
1775 nvme_dev_remove(dev);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001776 delete:
Matthew Wilcox740216f2011-02-15 16:28:20 -05001777 spin_lock(&dev_list_lock);
1778 list_del(&dev->node);
1779 spin_unlock(&dev_list_lock);
1780
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001781 nvme_free_queues(dev);
1782 unmap:
1783 iounmap(dev->bar);
Matthew Wilcox574e8b92011-02-01 16:24:35 -05001784 disable_msix:
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001785 pci_disable_msix(pdev);
1786 nvme_release_instance(dev);
Matthew Wilcox091b6092011-02-10 09:56:01 -05001787 nvme_release_prp_pools(dev);
Matthew Wilcox574e8b92011-02-01 16:24:35 -05001788 disable:
Shane Michael Matthews0ee5a7d2011-02-01 08:49:30 -05001789 pci_disable_device(pdev);
Matthew Wilcox574e8b92011-02-01 16:24:35 -05001790 pci_release_regions(pdev);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001791 free:
1792 kfree(dev->queues);
1793 kfree(dev->entry);
1794 kfree(dev);
1795 return result;
1796}
1797
Greg Kroah-Hartman8d85fce2012-12-21 15:13:49 -08001798static void nvme_remove(struct pci_dev *pdev)
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001799{
1800 struct nvme_dev *dev = pci_get_drvdata(pdev);
Keith Busch5e82e952013-02-19 10:17:58 -07001801 misc_deregister(&dev->miscdev);
1802 kref_put(&dev->kref, nvme_free_dev);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001803}
1804
1805/* These functions are yet to be implemented */
1806#define nvme_error_detected NULL
1807#define nvme_dump_registers NULL
1808#define nvme_link_reset NULL
1809#define nvme_slot_reset NULL
1810#define nvme_error_resume NULL
1811#define nvme_suspend NULL
1812#define nvme_resume NULL
1813
Stephen Hemminger1d352032012-09-07 09:33:17 -07001814static const struct pci_error_handlers nvme_err_handler = {
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001815 .error_detected = nvme_error_detected,
1816 .mmio_enabled = nvme_dump_registers,
1817 .link_reset = nvme_link_reset,
1818 .slot_reset = nvme_slot_reset,
1819 .resume = nvme_error_resume,
1820};
1821
1822/* Move to pci_ids.h later */
1823#define PCI_CLASS_STORAGE_EXPRESS 0x010802
1824
1825static DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = {
1826 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
1827 { 0, }
1828};
1829MODULE_DEVICE_TABLE(pci, nvme_id_table);
1830
1831static struct pci_driver nvme_driver = {
1832 .name = "nvme",
1833 .id_table = nvme_id_table,
1834 .probe = nvme_probe,
Greg Kroah-Hartman8d85fce2012-12-21 15:13:49 -08001835 .remove = nvme_remove,
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001836 .suspend = nvme_suspend,
1837 .resume = nvme_resume,
1838 .err_handler = &nvme_err_handler,
1839};
1840
1841static int __init nvme_init(void)
1842{
Matthew Wilcox0ac13142012-07-31 13:31:15 -04001843 int result;
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001844
1845 nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
1846 if (IS_ERR(nvme_thread))
1847 return PTR_ERR(nvme_thread);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001848
Keith Busch5c42ea12012-07-25 16:05:18 -06001849 result = register_blkdev(nvme_major, "nvme");
1850 if (result < 0)
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001851 goto kill_kthread;
Keith Busch5c42ea12012-07-25 16:05:18 -06001852 else if (result > 0)
Matthew Wilcox0ac13142012-07-31 13:31:15 -04001853 nvme_major = result;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001854
1855 result = pci_register_driver(&nvme_driver);
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001856 if (result)
1857 goto unregister_blkdev;
1858 return 0;
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001859
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001860 unregister_blkdev:
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001861 unregister_blkdev(nvme_major, "nvme");
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001862 kill_kthread:
1863 kthread_stop(nvme_thread);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001864 return result;
1865}
1866
1867static void __exit nvme_exit(void)
1868{
1869 pci_unregister_driver(&nvme_driver);
1870 unregister_blkdev(nvme_major, "nvme");
Matthew Wilcox1fa6aea2011-03-02 18:37:18 -05001871 kthread_stop(nvme_thread);
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001872}
1873
1874MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
1875MODULE_LICENSE("GPL");
Matthew Wilcox366e8212012-01-10 16:30:15 -05001876MODULE_VERSION("0.8");
Matthew Wilcoxb60503b2011-01-20 12:50:14 -05001877module_init(nvme_init);
1878module_exit(nvme_exit);