blob: 08afbc7a2bb87171a2c4255c74705d39dd7a2b94 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Network block device - make block devices work over TCP
3 *
4 * Note that you can not swap over this thing, yet. Seems to work but
5 * deadlocks sometimes - you can not swap over TCP in general.
6 *
Pavel Macheka2531292010-07-18 14:27:13 +02007 * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com>
9 *
Pavel Machekdbf492d2006-06-25 05:47:42 -070010 * This file is released under GPLv2 or later.
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 *
Pavel Machekdbf492d2006-06-25 05:47:42 -070012 * (part of code stolen from loop.c)
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 */
14
15#include <linux/major.h>
16
17#include <linux/blkdev.h>
18#include <linux/module.h>
19#include <linux/init.h>
20#include <linux/sched.h>
21#include <linux/fs.h>
22#include <linux/bio.h>
23#include <linux/stat.h>
24#include <linux/errno.h>
25#include <linux/file.h>
26#include <linux/ioctl.h>
Arnd Bergmann2a48fc02010-06-02 14:28:52 +020027#include <linux/mutex.h>
Herbert Xu4b2f0262006-01-06 00:09:47 -080028#include <linux/compiler.h>
29#include <linux/err.h>
30#include <linux/kernel.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090031#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032#include <net/sock.h>
Trond Myklebust91cf45f2007-11-12 18:10:39 -080033#include <linux/net.h>
Laurent Vivier48cf6062008-04-29 01:02:46 -070034#include <linux/kthread.h>
Markus Pargmannb9c495b2015-04-02 10:11:37 +020035#include <linux/types.h>
Markus Pargmann30d53d92015-08-17 08:20:06 +020036#include <linux/debugfs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#include <asm/uaccess.h>
39#include <asm/types.h>
40
41#include <linux/nbd.h>
42
Markus Pargmann13e71d62015-04-02 10:11:35 +020043struct nbd_device {
Markus Pargmann22d109c2015-08-17 08:20:09 +020044 u32 flags;
Markus Pargmann13e71d62015-04-02 10:11:35 +020045 struct socket * sock; /* If == NULL, device is not ready, yet */
46 int magic;
47
48 spinlock_t queue_lock;
49 struct list_head queue_head; /* Requests waiting result */
50 struct request *active_req;
51 wait_queue_head_t active_wq;
52 struct list_head waiting_queue; /* Requests to be sent */
53 wait_queue_head_t waiting_wq;
54
55 struct mutex tx_lock;
56 struct gendisk *disk;
57 int blksize;
Markus Pargmannb9c495b2015-04-02 10:11:37 +020058 loff_t bytesize;
Markus Pargmann13e71d62015-04-02 10:11:35 +020059 int xmit_timeout;
Markus Pargmann1f7b5cf2015-10-29 12:01:34 +010060 bool timedout;
Markus Pargmann696697c2015-08-17 08:20:07 +020061 bool disconnect; /* a disconnect has been requested by user */
Markus Pargmann7e2893a2015-08-17 08:20:00 +020062
63 struct timer_list timeout_timer;
Markus Pargmann23272a672015-10-29 11:51:16 +010064 /* protects initialization and shutdown of the socket */
65 spinlock_t sock_lock;
Markus Pargmann7e2893a2015-08-17 08:20:00 +020066 struct task_struct *task_recv;
67 struct task_struct *task_send;
Markus Pargmann30d53d92015-08-17 08:20:06 +020068
69#if IS_ENABLED(CONFIG_DEBUG_FS)
70 struct dentry *dbg_dir;
71#endif
Markus Pargmann13e71d62015-04-02 10:11:35 +020072};
73
Markus Pargmann30d53d92015-08-17 08:20:06 +020074#if IS_ENABLED(CONFIG_DEBUG_FS)
75static struct dentry *nbd_dbg_dir;
76#endif
77
78#define nbd_name(nbd) ((nbd)->disk->disk_name)
79
Wanlong Gaof4507162012-03-28 14:42:51 -070080#define NBD_MAGIC 0x68797548
Linus Torvalds1da177e2005-04-16 15:20:36 -070081
Ingo van Lil9c7a4162006-07-01 04:36:36 -070082static unsigned int nbds_max = 16;
Paul Clements20a81432008-02-08 04:21:51 -080083static struct nbd_device *nbd_dev;
Laurent Vivierd71a6d72008-04-29 01:02:51 -070084static int max_part;
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
86/*
87 * Use just one lock (or at most 1 per NIC). Two arguments for this:
88 * 1. Each NIC is essentially a synchronization point for all servers
89 * accessed through that NIC so there's no need to have more locks
90 * than NICs anyway.
91 * 2. More locks lead to more "Dirty cache line bouncing" which will slow
92 * down each lock to the point where they're actually slower than just
93 * a single lock.
94 * Thanks go to Jens Axboe and Al Viro for their LKML emails explaining this!
95 */
96static DEFINE_SPINLOCK(nbd_lock);
97
Markus Pargmannd18509f2015-04-02 10:11:38 +020098static inline struct device *nbd_to_dev(struct nbd_device *nbd)
Linus Torvalds1da177e2005-04-16 15:20:36 -070099{
Markus Pargmannd18509f2015-04-02 10:11:38 +0200100 return disk_to_dev(nbd->disk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101}
102
Markus Pargmann37091fd2015-07-27 07:36:49 +0200103static bool nbd_is_connected(struct nbd_device *nbd)
104{
105 return !!nbd->task_recv;
106}
107
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108static const char *nbdcmd_to_ascii(int cmd)
109{
110 switch (cmd) {
111 case NBD_CMD_READ: return "read";
112 case NBD_CMD_WRITE: return "write";
113 case NBD_CMD_DISC: return "disconnect";
Alex Bligh75f187a2013-02-27 17:05:23 -0800114 case NBD_CMD_FLUSH: return "flush";
Paul Clementsa336d292012-10-04 17:16:18 -0700115 case NBD_CMD_TRIM: return "trim/discard";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 }
117 return "invalid";
118}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
Markus Pargmann37091fd2015-07-27 07:36:49 +0200120static int nbd_size_clear(struct nbd_device *nbd, struct block_device *bdev)
121{
122 bdev->bd_inode->i_size = 0;
123 set_capacity(nbd->disk, 0);
124 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);
125
126 return 0;
127}
128
129static void nbd_size_update(struct nbd_device *nbd, struct block_device *bdev)
130{
131 if (!nbd_is_connected(nbd))
132 return;
133
134 bdev->bd_inode->i_size = nbd->bytesize;
135 set_capacity(nbd->disk, nbd->bytesize >> 9);
136 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);
137}
138
139static int nbd_size_set(struct nbd_device *nbd, struct block_device *bdev,
140 int blocksize, int nr_blocks)
141{
142 int ret;
143
144 ret = set_blocksize(bdev, blocksize);
145 if (ret)
146 return ret;
147
148 nbd->blksize = blocksize;
149 nbd->bytesize = (loff_t)blocksize * (loff_t)nr_blocks;
150
151 nbd_size_update(nbd, bdev);
152
153 return 0;
154}
155
Markus Pargmannd18509f2015-04-02 10:11:38 +0200156static void nbd_end_request(struct nbd_device *nbd, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157{
Kiyoshi Ueda097c94a2007-12-11 17:44:06 -0500158 int error = req->errors ? -EIO : 0;
Jens Axboe165125e2007-07-24 09:28:11 +0200159 struct request_queue *q = req->q;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 unsigned long flags;
161
Markus Pargmannd18509f2015-04-02 10:11:38 +0200162 dev_dbg(nbd_to_dev(nbd), "request %p: %s\n", req,
163 error ? "failed" : "done");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164
165 spin_lock_irqsave(q->queue_lock, flags);
Tejun Heo1011c1b2009-05-07 22:24:45 +0900166 __blk_end_request_all(req, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 spin_unlock_irqrestore(q->queue_lock, flags);
168}
169
Markus Pargmanne018e752015-04-02 10:11:39 +0200170/*
171 * Forcibly shutdown the socket causing all listeners to error
172 */
Markus Pargmann36e47be2015-08-17 08:20:01 +0200173static void sock_shutdown(struct nbd_device *nbd)
Paul Clements7fdfd402007-10-16 23:27:37 -0700174{
Markus Pargmann23272a672015-10-29 11:51:16 +0100175 spin_lock_irq(&nbd->sock_lock);
176
177 if (!nbd->sock) {
178 spin_unlock_irq(&nbd->sock_lock);
Markus Pargmann260bbce2015-08-17 08:20:02 +0200179 return;
Markus Pargmann23272a672015-10-29 11:51:16 +0100180 }
Markus Pargmann260bbce2015-08-17 08:20:02 +0200181
182 dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n");
183 kernel_sock_shutdown(nbd->sock, SHUT_RDWR);
Markus Pargmann23272a672015-10-29 11:51:16 +0100184 sockfd_put(nbd->sock);
Markus Pargmann260bbce2015-08-17 08:20:02 +0200185 nbd->sock = NULL;
Markus Pargmann23272a672015-10-29 11:51:16 +0100186 spin_unlock_irq(&nbd->sock_lock);
187
188 del_timer(&nbd->timeout_timer);
Paul Clements7fdfd402007-10-16 23:27:37 -0700189}
190
191static void nbd_xmit_timeout(unsigned long arg)
192{
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200193 struct nbd_device *nbd = (struct nbd_device *)arg;
Markus Pargmanndcc909d2015-10-06 20:03:54 +0200194 unsigned long flags;
Paul Clements7fdfd402007-10-16 23:27:37 -0700195
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200196 if (list_empty(&nbd->queue_head))
197 return;
198
Markus Pargmann23272a672015-10-29 11:51:16 +0100199 spin_lock_irqsave(&nbd->sock_lock, flags);
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200200
Markus Pargmann1f7b5cf2015-10-29 12:01:34 +0100201 nbd->timedout = true;
Markus Pargmanndcc909d2015-10-06 20:03:54 +0200202
Markus Pargmann23272a672015-10-29 11:51:16 +0100203 if (nbd->sock)
204 kernel_sock_shutdown(nbd->sock, SHUT_RDWR);
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200205
Markus Pargmann23272a672015-10-29 11:51:16 +0100206 spin_unlock_irqrestore(&nbd->sock_lock, flags);
Markus Pargmanndcc909d2015-10-06 20:03:54 +0200207
Markus Pargmann23272a672015-10-29 11:51:16 +0100208 dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n");
Paul Clements7fdfd402007-10-16 23:27:37 -0700209}
210
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211/*
212 * Send or receive packet.
213 */
Wanlong Gaof4507162012-03-28 14:42:51 -0700214static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 int msg_flags)
216{
Wanlong Gaof4507162012-03-28 14:42:51 -0700217 struct socket *sock = nbd->sock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218 int result;
219 struct msghdr msg;
220 struct kvec iov;
Mel Gorman7f338fe2012-07-31 16:44:32 -0700221 unsigned long pflags = current->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
Mike Snitzerffc41cf2008-04-02 13:04:47 -0700223 if (unlikely(!sock)) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700224 dev_err(disk_to_dev(nbd->disk),
WANG Cong7f1b90f2011-08-19 14:48:22 +0200225 "Attempted %s on closed socket in sock_xmit\n",
226 (send ? "send" : "recv"));
Mike Snitzerffc41cf2008-04-02 13:04:47 -0700227 return -EINVAL;
228 }
229
Mel Gorman7f338fe2012-07-31 16:44:32 -0700230 current->flags |= PF_MEMALLOC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 do {
Mel Gorman7f338fe2012-07-31 16:44:32 -0700232 sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233 iov.iov_base = buf;
234 iov.iov_len = size;
235 msg.msg_name = NULL;
236 msg.msg_namelen = 0;
237 msg.msg_control = NULL;
238 msg.msg_controllen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 msg.msg_flags = msg_flags | MSG_NOSIGNAL;
240
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200241 if (send)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 result = kernel_sendmsg(sock, &msg, &iov, 1, size);
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200243 else
Namhyung Kim35fbf5b2011-05-28 14:44:46 +0200244 result = kernel_recvmsg(sock, &msg, &iov, 1, size,
245 msg.msg_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 if (result <= 0) {
248 if (result == 0)
249 result = -EPIPE; /* short read */
250 break;
251 }
252 size -= result;
253 buf += result;
254 } while (size > 0);
255
Mel Gorman7f338fe2012-07-31 16:44:32 -0700256 tsk_restore_flags(current, pflags, PF_MEMALLOC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200258 if (!send && nbd->xmit_timeout)
259 mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout);
260
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 return result;
262}
263
Wanlong Gaof4507162012-03-28 14:42:51 -0700264static inline int sock_send_bvec(struct nbd_device *nbd, struct bio_vec *bvec,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 int flags)
266{
267 int result;
268 void *kaddr = kmap(bvec->bv_page);
Wanlong Gaof4507162012-03-28 14:42:51 -0700269 result = sock_xmit(nbd, 1, kaddr + bvec->bv_offset,
270 bvec->bv_len, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 kunmap(bvec->bv_page);
272 return result;
273}
274
Paul Clements7fdfd402007-10-16 23:27:37 -0700275/* always call with the tx_lock held */
Wanlong Gaof4507162012-03-28 14:42:51 -0700276static int nbd_send_req(struct nbd_device *nbd, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277{
NeilBrown5705f702007-09-25 12:35:59 +0200278 int result, flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 struct nbd_request request;
Tejun Heo1011c1b2009-05-07 22:24:45 +0900280 unsigned long size = blk_rq_bytes(req);
Christoph Hellwig9dc6c802015-04-17 22:37:21 +0200281 u32 type;
282
283 if (req->cmd_type == REQ_TYPE_DRV_PRIV)
284 type = NBD_CMD_DISC;
285 else if (req->cmd_flags & REQ_DISCARD)
286 type = NBD_CMD_TRIM;
287 else if (req->cmd_flags & REQ_FLUSH)
288 type = NBD_CMD_FLUSH;
289 else if (rq_data_dir(req) == WRITE)
290 type = NBD_CMD_WRITE;
291 else
292 type = NBD_CMD_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293
Hani Benhabiles04cfac42014-06-06 14:38:30 -0700294 memset(&request, 0, sizeof(request));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 request.magic = htonl(NBD_REQUEST_MAGIC);
Christoph Hellwig9dc6c802015-04-17 22:37:21 +0200296 request.type = htonl(type);
297 if (type != NBD_CMD_FLUSH && type != NBD_CMD_DISC) {
Alex Bligh75f187a2013-02-27 17:05:23 -0800298 request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);
299 request.len = htonl(size);
300 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 memcpy(request.handle, &req, sizeof(req));
302
Markus Pargmannd18509f2015-04-02 10:11:38 +0200303 dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",
Christoph Hellwig9dc6c802015-04-17 22:37:21 +0200304 req, nbdcmd_to_ascii(type),
Markus Pargmannd18509f2015-04-02 10:11:38 +0200305 (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));
Wanlong Gaof4507162012-03-28 14:42:51 -0700306 result = sock_xmit(nbd, 1, &request, sizeof(request),
Christoph Hellwig9dc6c802015-04-17 22:37:21 +0200307 (type == NBD_CMD_WRITE) ? MSG_MORE : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 if (result <= 0) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700309 dev_err(disk_to_dev(nbd->disk),
WANG Cong7f1b90f2011-08-19 14:48:22 +0200310 "Send control failed (result %d)\n", result);
Markus Pargmanndab53132015-04-02 10:11:40 +0200311 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 }
313
Christoph Hellwig9dc6c802015-04-17 22:37:21 +0200314 if (type == NBD_CMD_WRITE) {
NeilBrown5705f702007-09-25 12:35:59 +0200315 struct req_iterator iter;
Kent Overstreet79886132013-11-23 17:19:00 -0800316 struct bio_vec bvec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 /*
318 * we are really probing at internals to determine
319 * whether to set MSG_MORE or not...
320 */
NeilBrown5705f702007-09-25 12:35:59 +0200321 rq_for_each_segment(bvec, req, iter) {
Jens Axboe6c92e692007-08-16 13:43:12 +0200322 flags = 0;
Kent Overstreet4550dd62013-08-07 14:26:21 -0700323 if (!rq_iter_last(bvec, iter))
Jens Axboe6c92e692007-08-16 13:43:12 +0200324 flags = MSG_MORE;
Markus Pargmannd18509f2015-04-02 10:11:38 +0200325 dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n",
326 req, bvec.bv_len);
Kent Overstreet79886132013-11-23 17:19:00 -0800327 result = sock_send_bvec(nbd, &bvec, flags);
Jens Axboe6c92e692007-08-16 13:43:12 +0200328 if (result <= 0) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700329 dev_err(disk_to_dev(nbd->disk),
WANG Cong7f1b90f2011-08-19 14:48:22 +0200330 "Send data failed (result %d)\n",
331 result);
Markus Pargmanndab53132015-04-02 10:11:40 +0200332 return -EIO;
Jens Axboe6c92e692007-08-16 13:43:12 +0200333 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334 }
335 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337}
338
Wanlong Gaof4507162012-03-28 14:42:51 -0700339static struct request *nbd_find_request(struct nbd_device *nbd,
Denis Cheng0cbc591b2007-10-16 23:26:14 -0700340 struct request *xreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341{
Denis Chengd2c97402007-10-16 23:26:14 -0700342 struct request *req, *tmp;
Herbert Xu4b2f0262006-01-06 00:09:47 -0800343 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344
Wanlong Gaof4507162012-03-28 14:42:51 -0700345 err = wait_event_interruptible(nbd->active_wq, nbd->active_req != xreq);
Herbert Xu4b2f0262006-01-06 00:09:47 -0800346 if (unlikely(err))
Markus Pargmannde9ad6d2015-04-02 10:11:41 +0200347 return ERR_PTR(err);
Herbert Xu4b2f0262006-01-06 00:09:47 -0800348
Wanlong Gaof4507162012-03-28 14:42:51 -0700349 spin_lock(&nbd->queue_lock);
350 list_for_each_entry_safe(req, tmp, &nbd->queue_head, queuelist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 if (req != xreq)
352 continue;
353 list_del_init(&req->queuelist);
Wanlong Gaof4507162012-03-28 14:42:51 -0700354 spin_unlock(&nbd->queue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 return req;
356 }
Wanlong Gaof4507162012-03-28 14:42:51 -0700357 spin_unlock(&nbd->queue_lock);
Herbert Xu4b2f0262006-01-06 00:09:47 -0800358
Markus Pargmannde9ad6d2015-04-02 10:11:41 +0200359 return ERR_PTR(-ENOENT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360}
361
Wanlong Gaof4507162012-03-28 14:42:51 -0700362static inline int sock_recv_bvec(struct nbd_device *nbd, struct bio_vec *bvec)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363{
364 int result;
365 void *kaddr = kmap(bvec->bv_page);
Wanlong Gaof4507162012-03-28 14:42:51 -0700366 result = sock_xmit(nbd, 0, kaddr + bvec->bv_offset, bvec->bv_len,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 MSG_WAITALL);
368 kunmap(bvec->bv_page);
369 return result;
370}
371
372/* NULL returned = something went wrong, inform userspace */
Wanlong Gaof4507162012-03-28 14:42:51 -0700373static struct request *nbd_read_stat(struct nbd_device *nbd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374{
375 int result;
376 struct nbd_reply reply;
377 struct request *req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378
379 reply.magic = 0;
Wanlong Gaof4507162012-03-28 14:42:51 -0700380 result = sock_xmit(nbd, 0, &reply, sizeof(reply), MSG_WAITALL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 if (result <= 0) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700382 dev_err(disk_to_dev(nbd->disk),
WANG Cong7f1b90f2011-08-19 14:48:22 +0200383 "Receive control failed (result %d)\n", result);
Markus Pargmann19391832015-08-17 08:20:03 +0200384 return ERR_PTR(result);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 }
Michal Feixe4b57e02006-07-30 03:03:31 -0700386
387 if (ntohl(reply.magic) != NBD_REPLY_MAGIC) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700388 dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n",
Michal Feixe4b57e02006-07-30 03:03:31 -0700389 (unsigned long)ntohl(reply.magic));
Markus Pargmann19391832015-08-17 08:20:03 +0200390 return ERR_PTR(-EPROTO);
Michal Feixe4b57e02006-07-30 03:03:31 -0700391 }
392
Wanlong Gaof4507162012-03-28 14:42:51 -0700393 req = nbd_find_request(nbd, *(struct request **)reply.handle);
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -0700394 if (IS_ERR(req)) {
Herbert Xu4b2f0262006-01-06 00:09:47 -0800395 result = PTR_ERR(req);
396 if (result != -ENOENT)
Markus Pargmann19391832015-08-17 08:20:03 +0200397 return ERR_PTR(result);
Herbert Xu4b2f0262006-01-06 00:09:47 -0800398
Wanlong Gaof4507162012-03-28 14:42:51 -0700399 dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%p)\n",
WANG Cong7f1b90f2011-08-19 14:48:22 +0200400 reply.handle);
Markus Pargmann19391832015-08-17 08:20:03 +0200401 return ERR_PTR(-EBADR);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402 }
403
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 if (ntohl(reply.error)) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700405 dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n",
WANG Cong7f1b90f2011-08-19 14:48:22 +0200406 ntohl(reply.error));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 req->errors++;
408 return req;
409 }
410
Markus Pargmannd18509f2015-04-02 10:11:38 +0200411 dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", req);
Christoph Hellwig9dc6c802015-04-17 22:37:21 +0200412 if (rq_data_dir(req) != WRITE) {
NeilBrown5705f702007-09-25 12:35:59 +0200413 struct req_iterator iter;
Kent Overstreet79886132013-11-23 17:19:00 -0800414 struct bio_vec bvec;
NeilBrown5705f702007-09-25 12:35:59 +0200415
416 rq_for_each_segment(bvec, req, iter) {
Kent Overstreet79886132013-11-23 17:19:00 -0800417 result = sock_recv_bvec(nbd, &bvec);
Jens Axboe6c92e692007-08-16 13:43:12 +0200418 if (result <= 0) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700419 dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n",
WANG Cong7f1b90f2011-08-19 14:48:22 +0200420 result);
Jens Axboe6c92e692007-08-16 13:43:12 +0200421 req->errors++;
422 return req;
423 }
Markus Pargmannd18509f2015-04-02 10:11:38 +0200424 dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n",
425 req, bvec.bv_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 }
427 }
428 return req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429}
430
Kay Sieversedfaa7c2007-05-21 22:08:01 +0200431static ssize_t pid_show(struct device *dev,
432 struct device_attribute *attr, char *buf)
Paul Clements6b39bb62006-12-06 20:40:53 -0800433{
Kay Sieversedfaa7c2007-05-21 22:08:01 +0200434 struct gendisk *disk = dev_to_disk(dev);
Markus Pargmann6521d392015-08-17 08:20:05 +0200435 struct nbd_device *nbd = (struct nbd_device *)disk->private_data;
Kay Sieversedfaa7c2007-05-21 22:08:01 +0200436
Markus Pargmann6521d392015-08-17 08:20:05 +0200437 return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv));
Paul Clements6b39bb62006-12-06 20:40:53 -0800438}
439
Kay Sieversedfaa7c2007-05-21 22:08:01 +0200440static struct device_attribute pid_attr = {
Parag Warudkar01e8ef12008-10-18 20:28:50 -0700441 .attr = { .name = "pid", .mode = S_IRUGO},
Paul Clements6b39bb62006-12-06 20:40:53 -0800442 .show = pid_show,
443};
444
Markus Pargmann37091fd2015-07-27 07:36:49 +0200445static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446{
447 struct request *req;
WANG Cong84963042007-05-09 02:33:36 -0700448 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449
Wanlong Gaof4507162012-03-28 14:42:51 -0700450 BUG_ON(nbd->magic != NBD_MAGIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451
Mel Gorman7f338fe2012-07-31 16:44:32 -0700452 sk_set_memalloc(nbd->sock->sk);
Markus Pargmann6521d392015-08-17 08:20:05 +0200453
454 nbd->task_recv = current;
455
Wanlong Gaof4507162012-03-28 14:42:51 -0700456 ret = device_create_file(disk_to_dev(nbd->disk), &pid_attr);
WANG Cong84963042007-05-09 02:33:36 -0700457 if (ret) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700458 dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n");
Markus Pargmanndcc909d2015-10-06 20:03:54 +0200459
Markus Pargmann6521d392015-08-17 08:20:05 +0200460 nbd->task_recv = NULL;
Markus Pargmanndcc909d2015-10-06 20:03:54 +0200461
WANG Cong84963042007-05-09 02:33:36 -0700462 return ret;
463 }
Paul Clements6b39bb62006-12-06 20:40:53 -0800464
Markus Pargmann37091fd2015-07-27 07:36:49 +0200465 nbd_size_update(nbd, bdev);
466
Markus Pargmann19391832015-08-17 08:20:03 +0200467 while (1) {
468 req = nbd_read_stat(nbd);
469 if (IS_ERR(req)) {
470 ret = PTR_ERR(req);
471 break;
472 }
473
Markus Pargmannd18509f2015-04-02 10:11:38 +0200474 nbd_end_request(nbd, req);
Markus Pargmann19391832015-08-17 08:20:03 +0200475 }
Paul Clements6b39bb62006-12-06 20:40:53 -0800476
Markus Pargmann37091fd2015-07-27 07:36:49 +0200477 nbd_size_clear(nbd, bdev);
478
Markus Pargmann6521d392015-08-17 08:20:05 +0200479 device_remove_file(disk_to_dev(nbd->disk), &pid_attr);
480
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200481 nbd->task_recv = NULL;
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200482
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200483 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484}
485
Wanlong Gaof4507162012-03-28 14:42:51 -0700486static void nbd_clear_que(struct nbd_device *nbd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487{
488 struct request *req;
489
Wanlong Gaof4507162012-03-28 14:42:51 -0700490 BUG_ON(nbd->magic != NBD_MAGIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491
Herbert Xu4b2f0262006-01-06 00:09:47 -0800492 /*
Wanlong Gaof4507162012-03-28 14:42:51 -0700493 * Because we have set nbd->sock to NULL under the tx_lock, all
Herbert Xu4b2f0262006-01-06 00:09:47 -0800494 * modifications to the list must have completed by now. For
495 * the same reason, the active_req must be NULL.
496 *
497 * As a consequence, we don't need to take the spin lock while
498 * purging the list here.
499 */
Wanlong Gaof4507162012-03-28 14:42:51 -0700500 BUG_ON(nbd->sock);
501 BUG_ON(nbd->active_req);
Herbert Xu4b2f0262006-01-06 00:09:47 -0800502
Wanlong Gaof4507162012-03-28 14:42:51 -0700503 while (!list_empty(&nbd->queue_head)) {
504 req = list_entry(nbd->queue_head.next, struct request,
Herbert Xu4b2f0262006-01-06 00:09:47 -0800505 queuelist);
506 list_del_init(&req->queuelist);
507 req->errors++;
Markus Pargmannd18509f2015-04-02 10:11:38 +0200508 nbd_end_request(nbd, req);
Herbert Xu4b2f0262006-01-06 00:09:47 -0800509 }
Paul Clementsfded4e02012-09-17 14:09:02 -0700510
511 while (!list_empty(&nbd->waiting_queue)) {
512 req = list_entry(nbd->waiting_queue.next, struct request,
513 queuelist);
514 list_del_init(&req->queuelist);
515 req->errors++;
Markus Pargmannd18509f2015-04-02 10:11:38 +0200516 nbd_end_request(nbd, req);
Paul Clementsfded4e02012-09-17 14:09:02 -0700517 }
Markus Pargmanne78273c2015-08-17 08:20:04 +0200518 dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519}
520
Paul Clements7fdfd402007-10-16 23:27:37 -0700521
Wanlong Gaof4507162012-03-28 14:42:51 -0700522static void nbd_handle_req(struct nbd_device *nbd, struct request *req)
Laurent Vivier48cf6062008-04-29 01:02:46 -0700523{
Christoph Hellwig33659eb2010-08-07 18:17:56 +0200524 if (req->cmd_type != REQ_TYPE_FS)
Laurent Vivier48cf6062008-04-29 01:02:46 -0700525 goto error_out;
526
Christoph Hellwig9dc6c802015-04-17 22:37:21 +0200527 if (rq_data_dir(req) == WRITE &&
528 (nbd->flags & NBD_FLAG_READ_ONLY)) {
529 dev_err(disk_to_dev(nbd->disk),
530 "Write on read-only\n");
531 goto error_out;
Alex Bligh75f187a2013-02-27 17:05:23 -0800532 }
533
Laurent Vivier48cf6062008-04-29 01:02:46 -0700534 req->errors = 0;
535
Wanlong Gaof4507162012-03-28 14:42:51 -0700536 mutex_lock(&nbd->tx_lock);
537 if (unlikely(!nbd->sock)) {
538 mutex_unlock(&nbd->tx_lock);
539 dev_err(disk_to_dev(nbd->disk),
WANG Cong7f1b90f2011-08-19 14:48:22 +0200540 "Attempted send on closed socket\n");
Pavel Machek15746fc2009-04-02 16:58:42 -0700541 goto error_out;
Laurent Vivier48cf6062008-04-29 01:02:46 -0700542 }
543
Wanlong Gaof4507162012-03-28 14:42:51 -0700544 nbd->active_req = req;
Laurent Vivier48cf6062008-04-29 01:02:46 -0700545
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200546 if (nbd->xmit_timeout && list_empty_careful(&nbd->queue_head))
547 mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout);
548
Wanlong Gaof4507162012-03-28 14:42:51 -0700549 if (nbd_send_req(nbd, req) != 0) {
550 dev_err(disk_to_dev(nbd->disk), "Request send failed\n");
Laurent Vivier48cf6062008-04-29 01:02:46 -0700551 req->errors++;
Markus Pargmannd18509f2015-04-02 10:11:38 +0200552 nbd_end_request(nbd, req);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700553 } else {
Wanlong Gaof4507162012-03-28 14:42:51 -0700554 spin_lock(&nbd->queue_lock);
Chetan Loke01ff5db2012-07-31 08:47:13 +0200555 list_add_tail(&req->queuelist, &nbd->queue_head);
Wanlong Gaof4507162012-03-28 14:42:51 -0700556 spin_unlock(&nbd->queue_lock);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700557 }
558
Wanlong Gaof4507162012-03-28 14:42:51 -0700559 nbd->active_req = NULL;
560 mutex_unlock(&nbd->tx_lock);
561 wake_up_all(&nbd->active_wq);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700562
563 return;
564
565error_out:
566 req->errors++;
Markus Pargmannd18509f2015-04-02 10:11:38 +0200567 nbd_end_request(nbd, req);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700568}
569
Markus Pargmanncad73b22015-08-17 08:20:08 +0200570static int nbd_thread_send(void *data)
Laurent Vivier48cf6062008-04-29 01:02:46 -0700571{
Wanlong Gaof4507162012-03-28 14:42:51 -0700572 struct nbd_device *nbd = data;
Laurent Vivier48cf6062008-04-29 01:02:46 -0700573 struct request *req;
574
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200575 nbd->task_send = current;
576
Dongsheng Yang8698a742014-03-11 18:09:12 +0800577 set_user_nice(current, MIN_NICE);
Wanlong Gaof4507162012-03-28 14:42:51 -0700578 while (!kthread_should_stop() || !list_empty(&nbd->waiting_queue)) {
Laurent Vivier48cf6062008-04-29 01:02:46 -0700579 /* wait for something to do */
Wanlong Gaof4507162012-03-28 14:42:51 -0700580 wait_event_interruptible(nbd->waiting_wq,
Laurent Vivier48cf6062008-04-29 01:02:46 -0700581 kthread_should_stop() ||
Wanlong Gaof4507162012-03-28 14:42:51 -0700582 !list_empty(&nbd->waiting_queue));
Laurent Vivier48cf6062008-04-29 01:02:46 -0700583
584 /* extract request */
Wanlong Gaof4507162012-03-28 14:42:51 -0700585 if (list_empty(&nbd->waiting_queue))
Laurent Vivier48cf6062008-04-29 01:02:46 -0700586 continue;
587
Wanlong Gaof4507162012-03-28 14:42:51 -0700588 spin_lock_irq(&nbd->queue_lock);
589 req = list_entry(nbd->waiting_queue.next, struct request,
Laurent Vivier48cf6062008-04-29 01:02:46 -0700590 queuelist);
591 list_del_init(&req->queuelist);
Wanlong Gaof4507162012-03-28 14:42:51 -0700592 spin_unlock_irq(&nbd->queue_lock);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700593
594 /* handle request */
Wanlong Gaof4507162012-03-28 14:42:51 -0700595 nbd_handle_req(nbd, req);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700596 }
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200597
598 nbd->task_send = NULL;
599
Laurent Vivier48cf6062008-04-29 01:02:46 -0700600 return 0;
601}
602
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603/*
604 * We always wait for result of write, for now. It would be nice to make it optional
605 * in future
Wanlong Gaof4507162012-03-28 14:42:51 -0700606 * if ((rq_data_dir(req) == WRITE) && (nbd->flags & NBD_WRITE_NOCHK))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 * { printk( "Warning: Ignoring result!\n"); nbd_end_request( req ); }
608 */
609
Markus Pargmanncad73b22015-08-17 08:20:08 +0200610static void nbd_request_handler(struct request_queue *q)
Alex Elder398eb082013-02-27 17:05:28 -0800611 __releases(q->queue_lock) __acquires(q->queue_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612{
613 struct request *req;
614
Tejun Heo9934c8c2009-05-08 11:54:16 +0900615 while ((req = blk_fetch_request(q)) != NULL) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700616 struct nbd_device *nbd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617
Laurent Vivier48cf6062008-04-29 01:02:46 -0700618 spin_unlock_irq(q->queue_lock);
619
Wanlong Gaof4507162012-03-28 14:42:51 -0700620 nbd = req->rq_disk->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621
Wanlong Gaof4507162012-03-28 14:42:51 -0700622 BUG_ON(nbd->magic != NBD_MAGIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623
Markus Pargmannd18509f2015-04-02 10:11:38 +0200624 dev_dbg(nbd_to_dev(nbd), "request %p: dequeued (flags=%x)\n",
625 req, req->cmd_type);
626
Wanlong Gaof4507162012-03-28 14:42:51 -0700627 if (unlikely(!nbd->sock)) {
Dan Streetmanda6ccaa2016-01-14 13:42:32 -0500628 dev_err_ratelimited(disk_to_dev(nbd->disk),
629 "Attempted send on closed socket\n");
Paul Clements4d48a542009-02-11 13:04:45 -0800630 req->errors++;
Markus Pargmannd18509f2015-04-02 10:11:38 +0200631 nbd_end_request(nbd, req);
Paul Clements4d48a542009-02-11 13:04:45 -0800632 spin_lock_irq(q->queue_lock);
633 continue;
634 }
635
Wanlong Gaof4507162012-03-28 14:42:51 -0700636 spin_lock_irq(&nbd->queue_lock);
637 list_add_tail(&req->queuelist, &nbd->waiting_queue);
638 spin_unlock_irq(&nbd->queue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639
Wanlong Gaof4507162012-03-28 14:42:51 -0700640 wake_up(&nbd->waiting_wq);
Herbert Xu4b2f0262006-01-06 00:09:47 -0800641
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 spin_lock_irq(q->queue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644}
645
Markus Pargmann23272a672015-10-29 11:51:16 +0100646static int nbd_set_socket(struct nbd_device *nbd, struct socket *sock)
647{
648 int ret = 0;
649
650 spin_lock_irq(&nbd->sock_lock);
651
652 if (nbd->sock) {
653 ret = -EBUSY;
654 goto out;
655 }
656
657 nbd->sock = sock;
658
659out:
660 spin_unlock_irq(&nbd->sock_lock);
661
662 return ret;
663}
664
Markus Pargmann0e4f0f62015-10-29 12:04:51 +0100665/* Reset all properties of an NBD device */
666static void nbd_reset(struct nbd_device *nbd)
667{
668 nbd->disconnect = false;
669 nbd->timedout = false;
670 nbd->blksize = 1024;
671 nbd->bytesize = 0;
672 set_capacity(nbd->disk, 0);
673 nbd->flags = 0;
674 nbd->xmit_timeout = 0;
675 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue);
676 del_timer_sync(&nbd->timeout_timer);
677}
678
679static void nbd_bdev_reset(struct block_device *bdev)
680{
681 set_device_ro(bdev, false);
682 bdev->bd_inode->i_size = 0;
683 if (max_part > 0) {
684 blkdev_reread_part(bdev);
685 bdev->bd_invalidated = 1;
686 }
687}
688
Markus Pargmannd02cf532015-10-29 12:06:15 +0100689static void nbd_parse_flags(struct nbd_device *nbd, struct block_device *bdev)
690{
691 if (nbd->flags & NBD_FLAG_READ_ONLY)
692 set_device_ro(bdev, true);
693 if (nbd->flags & NBD_FLAG_SEND_TRIM)
694 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue);
695 if (nbd->flags & NBD_FLAG_SEND_FLUSH)
696 blk_queue_flush(nbd->disk->queue, REQ_FLUSH);
697 else
698 blk_queue_flush(nbd->disk->queue, 0);
699}
700
Markus Pargmann30d53d92015-08-17 08:20:06 +0200701static int nbd_dev_dbg_init(struct nbd_device *nbd);
702static void nbd_dev_dbg_close(struct nbd_device *nbd);
703
Pavel Machek1a2ad212009-04-02 16:58:41 -0700704/* Must be called with tx_lock held */
705
Wanlong Gaof4507162012-03-28 14:42:51 -0700706static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
Pavel Machek1a2ad212009-04-02 16:58:41 -0700707 unsigned int cmd, unsigned long arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 switch (cmd) {
Pavel Machek1a2ad212009-04-02 16:58:41 -0700710 case NBD_DISCONNECT: {
711 struct request sreq;
712
Wanlong Gaof4507162012-03-28 14:42:51 -0700713 dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n");
Paolo Bonzini3a2d63f82013-02-27 17:05:25 -0800714 if (!nbd->sock)
715 return -EINVAL;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700716
Paolo Bonzini3a2d63f82013-02-27 17:05:25 -0800717 mutex_unlock(&nbd->tx_lock);
718 fsync_bdev(bdev);
719 mutex_lock(&nbd->tx_lock);
FUJITA Tomonori4f54eec2008-04-29 09:54:37 +0200720 blk_rq_init(NULL, &sreq);
Christoph Hellwig4f8c9512015-04-17 22:37:16 +0200721 sreq.cmd_type = REQ_TYPE_DRV_PRIV;
Paolo Bonzini3a2d63f82013-02-27 17:05:25 -0800722
723 /* Check again after getting mutex back. */
Wanlong Gaof4507162012-03-28 14:42:51 -0700724 if (!nbd->sock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725 return -EINVAL;
Paolo Bonzini3a2d63f82013-02-27 17:05:25 -0800726
Markus Pargmann696697c2015-08-17 08:20:07 +0200727 nbd->disconnect = true;
Paul Clementsc378f702013-07-03 15:09:04 -0700728
Wanlong Gaof4507162012-03-28 14:42:51 -0700729 nbd_send_req(nbd, &sreq);
Paul Clementsc378f702013-07-03 15:09:04 -0700730 return 0;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700731 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
Markus Pargmann23272a672015-10-29 11:51:16 +0100733 case NBD_CLEAR_SOCK:
734 sock_shutdown(nbd);
Wanlong Gaof4507162012-03-28 14:42:51 -0700735 nbd_clear_que(nbd);
736 BUG_ON(!list_empty(&nbd->queue_head));
Paul Clementsfded4e02012-09-17 14:09:02 -0700737 BUG_ON(!list_empty(&nbd->waiting_queue));
Paolo Bonzini3a2d63f82013-02-27 17:05:25 -0800738 kill_bdev(bdev);
Pavel Machek1a2ad212009-04-02 16:58:41 -0700739 return 0;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700740
741 case NBD_SET_SOCK: {
Al Viroe2511572014-03-05 20:41:36 -0500742 int err;
Markus Pargmann23272a672015-10-29 11:51:16 +0100743 struct socket *sock = sockfd_lookup(arg, &err);
744
745 if (!sock)
746 return err;
747
748 err = nbd_set_socket(nbd, sock);
749 if (!err && max_part)
750 bdev->bd_invalidated = 1;
751
752 return err;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700753 }
754
Markus Pargmann37091fd2015-07-27 07:36:49 +0200755 case NBD_SET_BLKSIZE: {
Arnd Bergmann5e454c62016-03-05 00:49:31 +0100756 loff_t bsize = div_s64(nbd->bytesize, arg);
Markus Pargmann37091fd2015-07-27 07:36:49 +0200757
758 return nbd_size_set(nbd, bdev, arg, bsize);
759 }
Pavel Machek1a2ad212009-04-02 16:58:41 -0700760
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 case NBD_SET_SIZE:
Markus Pargmann37091fd2015-07-27 07:36:49 +0200762 return nbd_size_set(nbd, bdev, nbd->blksize,
763 arg / nbd->blksize);
764
765 case NBD_SET_SIZE_BLOCKS:
766 return nbd_size_set(nbd, bdev, nbd->blksize, arg);
Pavel Machek1a2ad212009-04-02 16:58:41 -0700767
Paul Clements7fdfd402007-10-16 23:27:37 -0700768 case NBD_SET_TIMEOUT:
Wanlong Gaof4507162012-03-28 14:42:51 -0700769 nbd->xmit_timeout = arg * HZ;
Markus Pargmann7e2893a2015-08-17 08:20:00 +0200770 if (arg)
771 mod_timer(&nbd->timeout_timer,
772 jiffies + nbd->xmit_timeout);
773 else
774 del_timer_sync(&nbd->timeout_timer);
775
Paul Clements7fdfd402007-10-16 23:27:37 -0700776 return 0;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700777
Paul Clements2f012502012-10-04 17:16:15 -0700778 case NBD_SET_FLAGS:
779 nbd->flags = arg;
780 return 0;
781
Pavel Machek1a2ad212009-04-02 16:58:41 -0700782 case NBD_DO_IT: {
783 struct task_struct *thread;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700784 int error;
785
Markus Pargmann6521d392015-08-17 08:20:05 +0200786 if (nbd->task_recv)
Pavel Machekc91192d2009-01-15 13:51:03 -0800787 return -EBUSY;
Al Viroe2511572014-03-05 20:41:36 -0500788 if (!nbd->sock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 return -EINVAL;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700790
Wanlong Gaof4507162012-03-28 14:42:51 -0700791 mutex_unlock(&nbd->tx_lock);
Pavel Machek1a2ad212009-04-02 16:58:41 -0700792
Markus Pargmannd02cf532015-10-29 12:06:15 +0100793 nbd_parse_flags(nbd, bdev);
Paul Clementsa336d292012-10-04 17:16:18 -0700794
Markus Pargmanncad73b22015-08-17 08:20:08 +0200795 thread = kthread_run(nbd_thread_send, nbd, "%s",
Markus Pargmann30d53d92015-08-17 08:20:06 +0200796 nbd_name(nbd));
Pavel Machek1a2ad212009-04-02 16:58:41 -0700797 if (IS_ERR(thread)) {
Wanlong Gaof4507162012-03-28 14:42:51 -0700798 mutex_lock(&nbd->tx_lock);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700799 return PTR_ERR(thread);
Pavel Machek1a2ad212009-04-02 16:58:41 -0700800 }
Markus Pargmannd06df602015-04-02 10:11:36 +0200801
Markus Pargmann30d53d92015-08-17 08:20:06 +0200802 nbd_dev_dbg_init(nbd);
Markus Pargmann37091fd2015-07-27 07:36:49 +0200803 error = nbd_thread_recv(nbd, bdev);
Markus Pargmann30d53d92015-08-17 08:20:06 +0200804 nbd_dev_dbg_close(nbd);
Laurent Vivier48cf6062008-04-29 01:02:46 -0700805 kthread_stop(thread);
Pavel Machek1a2ad212009-04-02 16:58:41 -0700806
Wanlong Gaof4507162012-03-28 14:42:51 -0700807 mutex_lock(&nbd->tx_lock);
Markus Pargmann19391832015-08-17 08:20:03 +0200808
Markus Pargmann36e47be2015-08-17 08:20:01 +0200809 sock_shutdown(nbd);
Wanlong Gaof4507162012-03-28 14:42:51 -0700810 nbd_clear_que(nbd);
Paolo Bonzini3a2d63f82013-02-27 17:05:25 -0800811 kill_bdev(bdev);
Markus Pargmann0e4f0f62015-10-29 12:04:51 +0100812 nbd_bdev_reset(bdev);
813
Paul Clementsc378f702013-07-03 15:09:04 -0700814 if (nbd->disconnect) /* user requested, ignore socket errors */
Markus Pargmann1f7b5cf2015-10-29 12:01:34 +0100815 error = 0;
816 if (nbd->timedout)
817 error = -ETIMEDOUT;
818
Markus Pargmann0e4f0f62015-10-29 12:04:51 +0100819 nbd_reset(nbd);
820
Markus Pargmann19391832015-08-17 08:20:03 +0200821 return error;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700822 }
823
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824 case NBD_CLEAR_QUE:
Herbert Xu4b2f0262006-01-06 00:09:47 -0800825 /*
826 * This is for compatibility only. The queue is always cleared
827 * by NBD_DO_IT or NBD_CLEAR_SOCK.
828 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 return 0;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700830
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831 case NBD_PRINT_DEBUG:
Wanlong Gaof4507162012-03-28 14:42:51 -0700832 dev_info(disk_to_dev(nbd->disk),
WANG Cong5eedf542011-08-19 14:48:28 +0200833 "next = %p, prev = %p, head = %p\n",
Wanlong Gaof4507162012-03-28 14:42:51 -0700834 nbd->queue_head.next, nbd->queue_head.prev,
835 &nbd->queue_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 return 0;
837 }
Pavel Machek1a2ad212009-04-02 16:58:41 -0700838 return -ENOTTY;
839}
840
841static int nbd_ioctl(struct block_device *bdev, fmode_t mode,
842 unsigned int cmd, unsigned long arg)
843{
Wanlong Gaof4507162012-03-28 14:42:51 -0700844 struct nbd_device *nbd = bdev->bd_disk->private_data;
Pavel Machek1a2ad212009-04-02 16:58:41 -0700845 int error;
846
847 if (!capable(CAP_SYS_ADMIN))
848 return -EPERM;
849
Wanlong Gaof4507162012-03-28 14:42:51 -0700850 BUG_ON(nbd->magic != NBD_MAGIC);
Pavel Machek1a2ad212009-04-02 16:58:41 -0700851
Wanlong Gaof4507162012-03-28 14:42:51 -0700852 mutex_lock(&nbd->tx_lock);
853 error = __nbd_ioctl(bdev, nbd, cmd, arg);
854 mutex_unlock(&nbd->tx_lock);
Pavel Machek1a2ad212009-04-02 16:58:41 -0700855
856 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857}
858
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -0700859static const struct block_device_operations nbd_fops =
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860{
861 .owner = THIS_MODULE,
Arnd Bergmann8a6cfeb2010-07-08 10:18:46 +0200862 .ioctl = nbd_ioctl,
Al Viro263a3df2016-01-07 10:04:37 -0500863 .compat_ioctl = nbd_ioctl,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864};
865
Markus Pargmann30d53d92015-08-17 08:20:06 +0200866#if IS_ENABLED(CONFIG_DEBUG_FS)
867
868static int nbd_dbg_tasks_show(struct seq_file *s, void *unused)
869{
870 struct nbd_device *nbd = s->private;
871
872 if (nbd->task_recv)
873 seq_printf(s, "recv: %d\n", task_pid_nr(nbd->task_recv));
874 if (nbd->task_send)
875 seq_printf(s, "send: %d\n", task_pid_nr(nbd->task_send));
876
877 return 0;
878}
879
880static int nbd_dbg_tasks_open(struct inode *inode, struct file *file)
881{
882 return single_open(file, nbd_dbg_tasks_show, inode->i_private);
883}
884
885static const struct file_operations nbd_dbg_tasks_ops = {
886 .open = nbd_dbg_tasks_open,
887 .read = seq_read,
888 .llseek = seq_lseek,
889 .release = single_release,
890};
891
892static int nbd_dbg_flags_show(struct seq_file *s, void *unused)
893{
894 struct nbd_device *nbd = s->private;
895 u32 flags = nbd->flags;
896
897 seq_printf(s, "Hex: 0x%08x\n\n", flags);
898
899 seq_puts(s, "Known flags:\n");
900
901 if (flags & NBD_FLAG_HAS_FLAGS)
902 seq_puts(s, "NBD_FLAG_HAS_FLAGS\n");
903 if (flags & NBD_FLAG_READ_ONLY)
904 seq_puts(s, "NBD_FLAG_READ_ONLY\n");
905 if (flags & NBD_FLAG_SEND_FLUSH)
906 seq_puts(s, "NBD_FLAG_SEND_FLUSH\n");
907 if (flags & NBD_FLAG_SEND_TRIM)
908 seq_puts(s, "NBD_FLAG_SEND_TRIM\n");
909
910 return 0;
911}
912
913static int nbd_dbg_flags_open(struct inode *inode, struct file *file)
914{
915 return single_open(file, nbd_dbg_flags_show, inode->i_private);
916}
917
918static const struct file_operations nbd_dbg_flags_ops = {
919 .open = nbd_dbg_flags_open,
920 .read = seq_read,
921 .llseek = seq_lseek,
922 .release = single_release,
923};
924
925static int nbd_dev_dbg_init(struct nbd_device *nbd)
926{
927 struct dentry *dir;
Markus Pargmann27ea43f2015-10-24 21:15:34 +0200928
929 if (!nbd_dbg_dir)
930 return -EIO;
Markus Pargmann30d53d92015-08-17 08:20:06 +0200931
932 dir = debugfs_create_dir(nbd_name(nbd), nbd_dbg_dir);
Markus Pargmann27ea43f2015-10-24 21:15:34 +0200933 if (!dir) {
934 dev_err(nbd_to_dev(nbd), "Failed to create debugfs dir for '%s'\n",
935 nbd_name(nbd));
936 return -EIO;
Markus Pargmann30d53d92015-08-17 08:20:06 +0200937 }
938 nbd->dbg_dir = dir;
939
Markus Pargmann27ea43f2015-10-24 21:15:34 +0200940 debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_ops);
941 debugfs_create_u64("size_bytes", 0444, dir, &nbd->bytesize);
942 debugfs_create_u32("timeout", 0444, dir, &nbd->xmit_timeout);
943 debugfs_create_u32("blocksize", 0444, dir, &nbd->blksize);
944 debugfs_create_file("flags", 0444, dir, &nbd, &nbd_dbg_flags_ops);
Markus Pargmann30d53d92015-08-17 08:20:06 +0200945
946 return 0;
947}
948
949static void nbd_dev_dbg_close(struct nbd_device *nbd)
950{
951 debugfs_remove_recursive(nbd->dbg_dir);
952}
953
954static int nbd_dbg_init(void)
955{
956 struct dentry *dbg_dir;
957
958 dbg_dir = debugfs_create_dir("nbd", NULL);
Markus Pargmann27ea43f2015-10-24 21:15:34 +0200959 if (!dbg_dir)
960 return -EIO;
Markus Pargmann30d53d92015-08-17 08:20:06 +0200961
962 nbd_dbg_dir = dbg_dir;
963
964 return 0;
965}
966
967static void nbd_dbg_close(void)
968{
969 debugfs_remove_recursive(nbd_dbg_dir);
970}
971
972#else /* IS_ENABLED(CONFIG_DEBUG_FS) */
973
974static int nbd_dev_dbg_init(struct nbd_device *nbd)
975{
976 return 0;
977}
978
979static void nbd_dev_dbg_close(struct nbd_device *nbd)
980{
981}
982
983static int nbd_dbg_init(void)
984{
985 return 0;
986}
987
988static void nbd_dbg_close(void)
989{
990}
991
992#endif
993
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994/*
995 * And here should be modules and kernel interface
996 * (Just smiley confuses emacs :-)
997 */
998
999static int __init nbd_init(void)
1000{
1001 int err = -ENOMEM;
1002 int i;
Laurent Vivierd71a6d72008-04-29 01:02:51 -07001003 int part_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004
Adrian Bunk5b7b18c2006-03-25 03:07:04 -08001005 BUILD_BUG_ON(sizeof(struct nbd_request) != 28);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006
Laurent Vivierd71a6d72008-04-29 01:02:51 -07001007 if (max_part < 0) {
WANG Cong7742ce42011-08-19 14:48:28 +02001008 printk(KERN_ERR "nbd: max_part must be >= 0\n");
Laurent Vivierd71a6d72008-04-29 01:02:51 -07001009 return -EINVAL;
1010 }
1011
1012 part_shift = 0;
Namhyung Kim5988ce22011-05-28 14:44:46 +02001013 if (max_part > 0) {
Laurent Vivierd71a6d72008-04-29 01:02:51 -07001014 part_shift = fls(max_part);
1015
Namhyung Kim5988ce22011-05-28 14:44:46 +02001016 /*
1017 * Adjust max_part according to part_shift as it is exported
1018 * to user space so that user can know the max number of
1019 * partition kernel should be able to manage.
1020 *
1021 * Note that -1 is required because partition 0 is reserved
1022 * for the whole disk.
1023 */
1024 max_part = (1UL << part_shift) - 1;
1025 }
1026
Namhyung Kim3b271082011-05-28 14:44:46 +02001027 if ((1UL << part_shift) > DISK_MAX_PARTS)
1028 return -EINVAL;
1029
1030 if (nbds_max > 1UL << (MINORBITS - part_shift))
1031 return -EINVAL;
1032
Sudip Mukherjeeff6b8092015-01-27 18:08:22 +05301033 nbd_dev = kcalloc(nbds_max, sizeof(*nbd_dev), GFP_KERNEL);
1034 if (!nbd_dev)
1035 return -ENOMEM;
1036
Lars Marowsky-Bree40be0c22005-05-01 08:59:07 -07001037 for (i = 0; i < nbds_max; i++) {
Laurent Vivierd71a6d72008-04-29 01:02:51 -07001038 struct gendisk *disk = alloc_disk(1 << part_shift);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 if (!disk)
1040 goto out;
1041 nbd_dev[i].disk = disk;
1042 /*
1043 * The new linux 2.5 block layer implementation requires
1044 * every gendisk to have its very own request_queue struct.
1045 * These structs are big so we dynamically allocate them.
1046 */
Markus Pargmanncad73b22015-08-17 08:20:08 +02001047 disk->queue = blk_init_queue(nbd_request_handler, &nbd_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 if (!disk->queue) {
1049 put_disk(disk);
1050 goto out;
1051 }
Jens Axboe31dcfab2008-10-31 10:06:37 +01001052 /*
1053 * Tell the block layer that we are not a rotational device
1054 */
1055 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue);
Mike Snitzerb277da02014-10-04 10:55:32 -06001056 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue);
Paul Clementsa336d292012-10-04 17:16:18 -07001057 disk->queue->limits.discard_granularity = 512;
Jens Axboe2bb4cd52015-07-14 08:15:12 -06001058 blk_queue_max_discard_sectors(disk->queue, UINT_MAX);
Paul Clementsa336d292012-10-04 17:16:18 -07001059 disk->queue->limits.discard_zeroes_data = 0;
Michal Belczyk078be022013-04-30 15:28:28 -07001060 blk_queue_max_hw_sectors(disk->queue, 65536);
1061 disk->queue->limits.max_sectors = 256;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 }
1063
1064 if (register_blkdev(NBD_MAJOR, "nbd")) {
1065 err = -EIO;
1066 goto out;
1067 }
1068
1069 printk(KERN_INFO "nbd: registered device at major %d\n", NBD_MAJOR);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070
Markus Pargmann30d53d92015-08-17 08:20:06 +02001071 nbd_dbg_init();
1072
Lars Marowsky-Bree40be0c22005-05-01 08:59:07 -07001073 for (i = 0; i < nbds_max; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 struct gendisk *disk = nbd_dev[i].disk;
Wanlong Gaof4507162012-03-28 14:42:51 -07001075 nbd_dev[i].magic = NBD_MAGIC;
Laurent Vivier48cf6062008-04-29 01:02:46 -07001076 INIT_LIST_HEAD(&nbd_dev[i].waiting_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 spin_lock_init(&nbd_dev[i].queue_lock);
Markus Pargmann23272a672015-10-29 11:51:16 +01001078 spin_lock_init(&nbd_dev[i].sock_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 INIT_LIST_HEAD(&nbd_dev[i].queue_head);
Ingo Molnar82d4dc52006-03-23 03:00:38 -08001080 mutex_init(&nbd_dev[i].tx_lock);
Markus Pargmann7e2893a2015-08-17 08:20:00 +02001081 init_timer(&nbd_dev[i].timeout_timer);
1082 nbd_dev[i].timeout_timer.function = nbd_xmit_timeout;
1083 nbd_dev[i].timeout_timer.data = (unsigned long)&nbd_dev[i];
Herbert Xu4b2f0262006-01-06 00:09:47 -08001084 init_waitqueue_head(&nbd_dev[i].active_wq);
Laurent Vivier48cf6062008-04-29 01:02:46 -07001085 init_waitqueue_head(&nbd_dev[i].waiting_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086 disk->major = NBD_MAJOR;
Laurent Vivierd71a6d72008-04-29 01:02:51 -07001087 disk->first_minor = i << part_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088 disk->fops = &nbd_fops;
1089 disk->private_data = &nbd_dev[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 sprintf(disk->disk_name, "nbd%d", i);
Markus Pargmann0e4f0f62015-10-29 12:04:51 +01001091 nbd_reset(&nbd_dev[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 add_disk(disk);
1093 }
1094
1095 return 0;
1096out:
1097 while (i--) {
1098 blk_cleanup_queue(nbd_dev[i].disk->queue);
1099 put_disk(nbd_dev[i].disk);
1100 }
Sven Wegenerf3944d62008-08-20 14:09:07 -07001101 kfree(nbd_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 return err;
1103}
1104
1105static void __exit nbd_cleanup(void)
1106{
1107 int i;
Markus Pargmann30d53d92015-08-17 08:20:06 +02001108
1109 nbd_dbg_close();
1110
Lars Marowsky-Bree40be0c22005-05-01 08:59:07 -07001111 for (i = 0; i < nbds_max; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112 struct gendisk *disk = nbd_dev[i].disk;
Lars Marowsky-Bree40be0c22005-05-01 08:59:07 -07001113 nbd_dev[i].magic = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114 if (disk) {
1115 del_gendisk(disk);
1116 blk_cleanup_queue(disk->queue);
1117 put_disk(disk);
1118 }
1119 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 unregister_blkdev(NBD_MAJOR, "nbd");
Sven Wegenerf3944d62008-08-20 14:09:07 -07001121 kfree(nbd_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 printk(KERN_INFO "nbd: unregistered device at major %d\n", NBD_MAJOR);
1123}
1124
1125module_init(nbd_init);
1126module_exit(nbd_cleanup);
1127
1128MODULE_DESCRIPTION("Network Block Device");
1129MODULE_LICENSE("GPL");
1130
Lars Marowsky-Bree40be0c22005-05-01 08:59:07 -07001131module_param(nbds_max, int, 0444);
Laurent Vivierd71a6d72008-04-29 01:02:51 -07001132MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)");
1133module_param(max_part, int, 0444);
1134MODULE_PARM_DESC(max_part, "number of partitions per device (default: 0)");