blob: 3ddaa11584ef80bea8416f25cf99619cef307461 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
Mikulas Patocka586e80e2008-10-21 17:44:59 +01008#include <linux/device-mapper.h>
9
Mike Snitzerf4790822013-09-12 18:06:12 -040010#include "dm.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include "dm-path-selector.h"
Mike Andersonb15546f2007-10-19 22:48:02 +010012#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070013
Mike Snitzere5863d92014-12-17 21:08:12 -050014#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/ctype.h>
16#include <linux/init.h>
17#include <linux/mempool.h>
18#include <linux/module.h>
19#include <linux/pagemap.h>
20#include <linux/slab.h>
21#include <linux/time.h>
22#include <linux/workqueue.h>
Mikulas Patocka35991652012-06-03 00:29:58 +010023#include <linux/delay.h>
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070024#include <scsi/scsi_dh.h>
Arun Sharma600634972011-07-26 16:09:06 -070025#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
Alasdair G Kergon72d94862006-06-26 00:27:35 -070027#define DM_MSG_PREFIX "multipath"
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000028#define DM_PG_INIT_DELAY_MSECS 2000
29#define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31/* Path properties */
32struct pgpath {
33 struct list_head list;
34
35 struct priority_group *pg; /* Owning PG */
Kiyoshi Ueda66800732008-10-10 13:36:58 +010036 unsigned is_active; /* Path status */
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 unsigned fail_count; /* Cumulative failure count */
38
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -080039 struct dm_path path;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000040 struct delayed_work activate_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -070041};
42
43#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
44
45/*
46 * Paths are grouped into Priority Groups and numbered from 1 upwards.
47 * Each has a path selector which controls which path gets used.
48 */
49struct priority_group {
50 struct list_head list;
51
52 struct multipath *m; /* Owning multipath instance */
53 struct path_selector ps;
54
55 unsigned pg_num; /* Reference number */
56 unsigned bypassed; /* Temporarily bypass this PG? */
57
58 unsigned nr_pgpaths; /* Number of paths in PG */
59 struct list_head pgpaths;
60};
61
62/* Multipath context */
63struct multipath {
64 struct list_head list;
65 struct dm_target *ti;
66
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070067 const char *hw_handler_name;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -070068 char *hw_handler_params;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000069
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010070 spinlock_t lock;
71
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 unsigned nr_priority_groups;
73 struct list_head priority_groups;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000074
75 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
76
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 unsigned pg_init_required; /* pg_init needs calling? */
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -070078 unsigned pg_init_in_progress; /* Only one pg_init allowed at once */
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000079 unsigned pg_init_delay_retry; /* Delay pg_init retry? */
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
81 unsigned nr_valid_paths; /* Total number of usable paths */
82 struct pgpath *current_pgpath;
83 struct priority_group *current_pg;
84 struct priority_group *next_pg; /* Switch to this PG if set */
85 unsigned repeat_count; /* I/Os left before calling PS again */
86
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010087 unsigned queue_io:1; /* Must we queue all I/O? */
88 unsigned queue_if_no_path:1; /* Queue I/O if last path fails? */
89 unsigned saved_queue_if_no_path:1; /* Saved state during suspension */
Mike Snitzera58a9352012-07-27 15:08:04 +010090 unsigned retain_attached_hw_handler:1; /* If there's already a hw_handler present, don't change it. */
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +000091 unsigned pg_init_disabled:1; /* pg_init is not currently allowed */
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010092
Dave Wysochanskic9e45582007-10-19 22:47:53 +010093 unsigned pg_init_retries; /* Number of times to retry pg_init */
94 unsigned pg_init_count; /* Number of times pg_init called */
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000095 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 struct work_struct trigger_event;
98
99 /*
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100100 * We must use a mempool of dm_mpath_io structs so that we
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 * can resubmit bios on error.
102 */
103 mempool_t *mpio_pool;
Mike Anderson6380f262009-12-10 23:52:21 +0000104
105 struct mutex work_mutex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106};
107
108/*
109 * Context information attached to each bio we process.
110 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100111struct dm_mpath_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 struct pgpath *pgpath;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100113 size_t nr_bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114};
115
116typedef int (*action_fn) (struct pgpath *pgpath);
117
Christoph Lametere18b8902006-12-06 20:33:20 -0800118static struct kmem_cache *_mpio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700120static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
David Howellsc4028952006-11-22 14:57:56 +0000121static void trigger_event(struct work_struct *work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700122static void activate_path(struct work_struct *work);
Hannes Reineckee8099172014-02-28 15:33:44 +0100123static int __pgpath_busy(struct pgpath *pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124
125
126/*-----------------------------------------------
127 * Allocation routines
128 *-----------------------------------------------*/
129
130static struct pgpath *alloc_pgpath(void)
131{
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700132 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
Mike Anderson224cb3e2008-08-29 09:36:09 +0200134 if (pgpath) {
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100135 pgpath->is_active = 1;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000136 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
Mike Anderson224cb3e2008-08-29 09:36:09 +0200137 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
139 return pgpath;
140}
141
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100142static void free_pgpath(struct pgpath *pgpath)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143{
144 kfree(pgpath);
145}
146
147static struct priority_group *alloc_priority_group(void)
148{
149 struct priority_group *pg;
150
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700151 pg = kzalloc(sizeof(*pg), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700153 if (pg)
154 INIT_LIST_HEAD(&pg->pgpaths);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155
156 return pg;
157}
158
159static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
160{
161 struct pgpath *pgpath, *tmp;
162
163 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
164 list_del(&pgpath->list);
165 dm_put_device(ti, pgpath->path.dev);
166 free_pgpath(pgpath);
167 }
168}
169
170static void free_priority_group(struct priority_group *pg,
171 struct dm_target *ti)
172{
173 struct path_selector *ps = &pg->ps;
174
175 if (ps->type) {
176 ps->type->destroy(ps);
177 dm_put_path_selector(ps->type);
178 }
179
180 free_pgpaths(&pg->pgpaths, ti);
181 kfree(pg);
182}
183
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700184static struct multipath *alloc_multipath(struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185{
186 struct multipath *m;
Mike Snitzerf4790822013-09-12 18:06:12 -0400187 unsigned min_ios = dm_get_reserved_rq_based_ios();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700189 m = kzalloc(sizeof(*m), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 if (m) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 INIT_LIST_HEAD(&m->priority_groups);
192 spin_lock_init(&m->lock);
193 m->queue_io = 1;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000194 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
David Howellsc4028952006-11-22 14:57:56 +0000195 INIT_WORK(&m->trigger_event, trigger_event);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000196 init_waitqueue_head(&m->pg_init_wait);
Mike Anderson6380f262009-12-10 23:52:21 +0000197 mutex_init(&m->work_mutex);
Mike Snitzerf4790822013-09-12 18:06:12 -0400198 m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 if (!m->mpio_pool) {
200 kfree(m);
201 return NULL;
202 }
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700203 m->ti = ti;
204 ti->private = m;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 }
206
207 return m;
208}
209
210static void free_multipath(struct multipath *m)
211{
212 struct priority_group *pg, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213
214 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
215 list_del(&pg->list);
216 free_priority_group(pg, m->ti);
217 }
218
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700219 kfree(m->hw_handler_name);
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700220 kfree(m->hw_handler_params);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 mempool_destroy(m->mpio_pool);
222 kfree(m);
223}
224
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100225static int set_mapinfo(struct multipath *m, union map_info *info)
226{
227 struct dm_mpath_io *mpio;
228
229 mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
230 if (!mpio)
231 return -ENOMEM;
232
233 memset(mpio, 0, sizeof(*mpio));
234 info->ptr = mpio;
235
236 return 0;
237}
238
239static void clear_mapinfo(struct multipath *m, union map_info *info)
240{
241 struct dm_mpath_io *mpio = info->ptr;
242
243 info->ptr = NULL;
244 mempool_free(mpio, m->mpio_pool);
245}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246
247/*-----------------------------------------------
248 * Path selection
249 *-----------------------------------------------*/
250
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100251static int __pg_init_all_paths(struct multipath *m)
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000252{
253 struct pgpath *pgpath;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000254 unsigned long pg_init_delay = 0;
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000255
Hannes Reinecke17f4ff42014-02-28 15:33:42 +0100256 if (m->pg_init_in_progress || m->pg_init_disabled)
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100257 return 0;
Hannes Reinecke17f4ff42014-02-28 15:33:42 +0100258
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000259 m->pg_init_count++;
260 m->pg_init_required = 0;
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100261
262 /* Check here to reset pg_init_required */
263 if (!m->current_pg)
264 return 0;
265
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000266 if (m->pg_init_delay_retry)
267 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
268 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000269 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
270 /* Skip failed paths */
271 if (!pgpath->is_active)
272 continue;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000273 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
274 pg_init_delay))
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000275 m->pg_init_in_progress++;
276 }
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100277 return m->pg_init_in_progress;
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000278}
279
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
281{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 m->current_pg = pgpath->pg;
283
284 /* Must we initialise the PG first, and queue I/O till it's ready? */
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700285 if (m->hw_handler_name) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 m->pg_init_required = 1;
287 m->queue_io = 1;
288 } else {
289 m->pg_init_required = 0;
290 m->queue_io = 0;
291 }
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100292
293 m->pg_init_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294}
295
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100296static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
297 size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298{
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -0800299 struct dm_path *path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100301 path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 if (!path)
303 return -ENXIO;
304
305 m->current_pgpath = path_to_pgpath(path);
306
307 if (m->current_pg != pg)
308 __switch_pg(m, m->current_pgpath);
309
310 return 0;
311}
312
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100313static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314{
315 struct priority_group *pg;
316 unsigned bypassed = 1;
317
Benjamin Marzinski1f271972014-08-13 13:53:42 -0500318 if (!m->nr_valid_paths) {
319 m->queue_io = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320 goto failed;
Benjamin Marzinski1f271972014-08-13 13:53:42 -0500321 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322
323 /* Were we instructed to switch PG? */
324 if (m->next_pg) {
325 pg = m->next_pg;
326 m->next_pg = NULL;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100327 if (!__choose_path_in_pg(m, pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 return;
329 }
330
331 /* Don't change PG until it has no remaining paths */
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100332 if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 return;
334
335 /*
336 * Loop through priority groups until we find a valid path.
337 * First time we skip PGs marked 'bypassed'.
Mike Christief220fd42012-06-03 00:29:45 +0100338 * Second time we only try the ones we skipped, but set
339 * pg_init_delay_retry so we do not hammer controllers.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 */
341 do {
342 list_for_each_entry(pg, &m->priority_groups, list) {
343 if (pg->bypassed == bypassed)
344 continue;
Mike Christief220fd42012-06-03 00:29:45 +0100345 if (!__choose_path_in_pg(m, pg, nr_bytes)) {
346 if (!bypassed)
347 m->pg_init_delay_retry = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 return;
Mike Christief220fd42012-06-03 00:29:45 +0100349 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 }
351 } while (bypassed--);
352
353failed:
354 m->current_pgpath = NULL;
355 m->current_pg = NULL;
356}
357
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800358/*
359 * Check whether bios must be queued in the device-mapper core rather
360 * than here in the target.
361 *
362 * m->lock must be held on entry.
363 *
364 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
365 * same value then we are not between multipath_presuspend()
366 * and multipath_resume() calls and we have no need to check
367 * for the DMF_NOFLUSH_SUSPENDING flag.
368 */
369static int __must_push_back(struct multipath *m)
370{
Hannes Reineckee8099172014-02-28 15:33:44 +0100371 return (m->queue_if_no_path ||
372 (m->queue_if_no_path != m->saved_queue_if_no_path &&
373 dm_noflush_suspending(m->ti)));
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800374}
375
Hannes Reinecke36fcffc2014-02-28 15:33:47 +0100376/*
377 * Map cloned requests
378 */
Mike Snitzere5863d92014-12-17 21:08:12 -0500379static int __multipath_map(struct dm_target *ti, struct request *clone,
380 union map_info *map_context,
381 struct request *rq, struct request **__clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382{
Hannes Reinecke36fcffc2014-02-28 15:33:47 +0100383 struct multipath *m = (struct multipath *) ti->private;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100384 int r = DM_MAPIO_REQUEUE;
Mike Snitzere5863d92014-12-17 21:08:12 -0500385 size_t nr_bytes = clone ? blk_rq_bytes(clone) : blk_rq_bytes(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 struct pgpath *pgpath;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100387 struct block_device *bdev;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100388 struct dm_mpath_io *mpio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600390 spin_lock_irq(&m->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391
392 /* Do we need to select a new pgpath? */
393 if (!m->current_pgpath ||
394 (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100395 __choose_pgpath(m, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
397 pgpath = m->current_pgpath;
398
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100399 if (!pgpath) {
400 if (!__must_push_back(m))
401 r = -EIO; /* Failed */
402 goto out_unlock;
Mike Snitzer6afbc012014-07-08 11:55:09 -0400403 } else if (m->queue_io || m->pg_init_required) {
Hannes Reineckee3bde042014-02-28 15:33:46 +0100404 __pg_init_all_paths(m);
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100405 goto out_unlock;
406 }
Mike Snitzer6afbc012014-07-08 11:55:09 -0400407
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100408 if (set_mapinfo(m, map_context) < 0)
409 /* ENOMEM, requeue */
410 goto out_unlock;
411
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100412 mpio = map_context->ptr;
413 mpio->pgpath = pgpath;
414 mpio->nr_bytes = nr_bytes;
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600415
416 bdev = pgpath->path.dev->bdev;
417
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600418 spin_unlock_irq(&m->lock);
419
Mike Snitzere5863d92014-12-17 21:08:12 -0500420 if (clone) {
421 /* Old request-based interface: allocated clone is passed in */
422 clone->q = bdev_get_queue(bdev);
423 clone->rq_disk = bdev->bd_disk;
424 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
425 } else {
426 /* blk-mq request-based interface */
427 *__clone = blk_get_request(bdev_get_queue(bdev),
Mike Snitzer02233342015-03-10 23:49:26 -0400428 rq_data_dir(rq), GFP_ATOMIC);
Mike Snitzer4c6dd532015-05-27 15:23:56 -0400429 if (IS_ERR(*__clone)) {
Mike Snitzere5863d92014-12-17 21:08:12 -0500430 /* ENOMEM, requeue */
Mike Snitzer4c6dd532015-05-27 15:23:56 -0400431 clear_mapinfo(m, map_context);
Mike Snitzere5863d92014-12-17 21:08:12 -0500432 return r;
Mike Snitzer4c6dd532015-05-27 15:23:56 -0400433 }
Mike Snitzere5863d92014-12-17 21:08:12 -0500434 (*__clone)->bio = (*__clone)->biotail = NULL;
435 (*__clone)->rq_disk = bdev->bd_disk;
436 (*__clone)->cmd_flags |= REQ_FAILFAST_TRANSPORT;
437 }
438
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100439 if (pgpath->pg->ps.type->start_io)
440 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
441 &pgpath->path,
442 nr_bytes);
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600443 return DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444
Hannes Reineckee3bde042014-02-28 15:33:46 +0100445out_unlock:
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600446 spin_unlock_irq(&m->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447
448 return r;
449}
450
Mike Snitzere5863d92014-12-17 21:08:12 -0500451static int multipath_map(struct dm_target *ti, struct request *clone,
452 union map_info *map_context)
453{
454 return __multipath_map(ti, clone, map_context, NULL, NULL);
455}
456
457static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
458 union map_info *map_context,
459 struct request **clone)
460{
461 return __multipath_map(ti, NULL, map_context, rq, clone);
462}
463
464static void multipath_release_clone(struct request *clone)
465{
466 blk_put_request(clone);
467}
468
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469/*
470 * If we run out of usable paths, should we queue I/O or error it?
471 */
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700472static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
473 unsigned save_old_value)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474{
475 unsigned long flags;
476
477 spin_lock_irqsave(&m->lock, flags);
478
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700479 if (save_old_value)
480 m->saved_queue_if_no_path = m->queue_if_no_path;
481 else
482 m->saved_queue_if_no_path = queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 m->queue_if_no_path = queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 spin_unlock_irqrestore(&m->lock, flags);
485
Hannes Reinecke63d832c2014-05-26 14:45:39 +0200486 if (!queue_if_no_path)
487 dm_table_run_md_queue_async(m->ti->table);
488
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 return 0;
490}
491
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492/*
493 * An event is triggered whenever a path is taken out of use.
494 * Includes path failure and PG bypass.
495 */
David Howellsc4028952006-11-22 14:57:56 +0000496static void trigger_event(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497{
David Howellsc4028952006-11-22 14:57:56 +0000498 struct multipath *m =
499 container_of(work, struct multipath, trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500
501 dm_table_event(m->ti->table);
502}
503
504/*-----------------------------------------------------------------
505 * Constructor/argument parsing:
506 * <#multipath feature args> [<arg>]*
507 * <#hw_handler args> [hw_handler [<arg>]*]
508 * <#priority groups>
509 * <initial priority group>
510 * [<selector> <#selector args> [<arg>]*
511 * <#paths> <#per-path selector args>
512 * [<path> [<arg>]* ]+ ]+
513 *---------------------------------------------------------------*/
Mike Snitzer498f0102011-08-02 12:32:04 +0100514static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 struct dm_target *ti)
516{
517 int r;
518 struct path_selector_type *pst;
519 unsigned ps_argc;
520
Mike Snitzer498f0102011-08-02 12:32:04 +0100521 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700522 {0, 1024, "invalid number of path selector args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 };
524
Mike Snitzer498f0102011-08-02 12:32:04 +0100525 pst = dm_get_path_selector(dm_shift_arg(as));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 if (!pst) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700527 ti->error = "unknown path selector type";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 return -EINVAL;
529 }
530
Mike Snitzer498f0102011-08-02 12:32:04 +0100531 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100532 if (r) {
533 dm_put_path_selector(pst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 return -EINVAL;
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100535 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536
537 r = pst->create(&pg->ps, ps_argc, as->argv);
538 if (r) {
539 dm_put_path_selector(pst);
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700540 ti->error = "path selector constructor failed";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541 return r;
542 }
543
544 pg->ps.type = pst;
Mike Snitzer498f0102011-08-02 12:32:04 +0100545 dm_consume_args(as, ps_argc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546
547 return 0;
548}
549
Mike Snitzer498f0102011-08-02 12:32:04 +0100550static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 struct dm_target *ti)
552{
553 int r;
554 struct pgpath *p;
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700555 struct multipath *m = ti->private;
Mike Snitzera58a9352012-07-27 15:08:04 +0100556 struct request_queue *q = NULL;
557 const char *attached_handler_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558
559 /* we need at least a path arg */
560 if (as->argc < 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700561 ti->error = "no device given";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100562 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 }
564
565 p = alloc_pgpath();
566 if (!p)
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100567 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568
Mike Snitzer498f0102011-08-02 12:32:04 +0100569 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +0000570 &p->path.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700572 ti->error = "error getting device";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 goto bad;
574 }
575
Mike Snitzera58a9352012-07-27 15:08:04 +0100576 if (m->retain_attached_hw_handler || m->hw_handler_name)
577 q = bdev_get_queue(p->path.dev->bdev);
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100578
Mike Snitzera58a9352012-07-27 15:08:04 +0100579 if (m->retain_attached_hw_handler) {
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200580retain:
Mike Snitzera58a9352012-07-27 15:08:04 +0100581 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
582 if (attached_handler_name) {
583 /*
584 * Reset hw_handler_name to match the attached handler
585 * and clear any hw_handler_params associated with the
586 * ignored handler.
587 *
588 * NB. This modifies the table line to show the actual
589 * handler instead of the original table passed in.
590 */
591 kfree(m->hw_handler_name);
592 m->hw_handler_name = attached_handler_name;
593
594 kfree(m->hw_handler_params);
595 m->hw_handler_params = NULL;
596 }
597 }
598
599 if (m->hw_handler_name) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100600 r = scsi_dh_attach(q, m->hw_handler_name);
601 if (r == -EBUSY) {
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200602 char b[BDEVNAME_SIZE];
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100603
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200604 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
605 bdevname(p->path.dev->bdev, b));
606 goto retain;
607 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700608 if (r < 0) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100609 ti->error = "error attaching hardware handler";
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700610 dm_put_device(ti, p->path.dev);
611 goto bad;
612 }
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700613
614 if (m->hw_handler_params) {
615 r = scsi_dh_set_params(q, m->hw_handler_params);
616 if (r < 0) {
617 ti->error = "unable to set hardware "
618 "handler parameters";
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700619 dm_put_device(ti, p->path.dev);
620 goto bad;
621 }
622 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700623 }
624
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
626 if (r) {
627 dm_put_device(ti, p->path.dev);
628 goto bad;
629 }
630
631 return p;
632
633 bad:
634 free_pgpath(p);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100635 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636}
637
Mike Snitzer498f0102011-08-02 12:32:04 +0100638static struct priority_group *parse_priority_group(struct dm_arg_set *as,
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700639 struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640{
Mike Snitzer498f0102011-08-02 12:32:04 +0100641 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700642 {1, 1024, "invalid number of paths"},
643 {0, 1024, "invalid number of selector args"}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 };
645
646 int r;
Mike Snitzer498f0102011-08-02 12:32:04 +0100647 unsigned i, nr_selector_args, nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 struct priority_group *pg;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700649 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650
651 if (as->argc < 2) {
652 as->argc = 0;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100653 ti->error = "not enough priority group arguments";
654 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 }
656
657 pg = alloc_priority_group();
658 if (!pg) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700659 ti->error = "couldn't allocate priority group";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100660 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 }
662 pg->m = m;
663
664 r = parse_path_selector(as, pg, ti);
665 if (r)
666 goto bad;
667
668 /*
669 * read the paths
670 */
Mike Snitzer498f0102011-08-02 12:32:04 +0100671 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 if (r)
673 goto bad;
674
Mike Snitzer498f0102011-08-02 12:32:04 +0100675 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 if (r)
677 goto bad;
678
Mike Snitzer498f0102011-08-02 12:32:04 +0100679 nr_args = 1 + nr_selector_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 for (i = 0; i < pg->nr_pgpaths; i++) {
681 struct pgpath *pgpath;
Mike Snitzer498f0102011-08-02 12:32:04 +0100682 struct dm_arg_set path_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
Mike Snitzer498f0102011-08-02 12:32:04 +0100684 if (as->argc < nr_args) {
Mikulas Patocka148acff2008-07-21 12:00:30 +0100685 ti->error = "not enough path parameters";
Alasdair G Kergon6bbf79a2010-08-12 04:13:49 +0100686 r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 goto bad;
Mikulas Patocka148acff2008-07-21 12:00:30 +0100688 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689
Mike Snitzer498f0102011-08-02 12:32:04 +0100690 path_args.argc = nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 path_args.argv = as->argv;
692
693 pgpath = parse_path(&path_args, &pg->ps, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100694 if (IS_ERR(pgpath)) {
695 r = PTR_ERR(pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696 goto bad;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100697 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698
699 pgpath->pg = pg;
700 list_add_tail(&pgpath->list, &pg->pgpaths);
Mike Snitzer498f0102011-08-02 12:32:04 +0100701 dm_consume_args(as, nr_args);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 }
703
704 return pg;
705
706 bad:
707 free_priority_group(pg, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100708 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709}
710
Mike Snitzer498f0102011-08-02 12:32:04 +0100711static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713 unsigned hw_argc;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700714 int ret;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700715 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716
Mike Snitzer498f0102011-08-02 12:32:04 +0100717 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700718 {0, 1024, "invalid number of hardware handler args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 };
720
Mike Snitzer498f0102011-08-02 12:32:04 +0100721 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 return -EINVAL;
723
724 if (!hw_argc)
725 return 0;
726
Mike Snitzer498f0102011-08-02 12:32:04 +0100727 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
Chandra Seetharaman14e98c52008-11-13 23:39:06 +0000728
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700729 if (hw_argc > 1) {
730 char *p;
731 int i, j, len = 4;
732
733 for (i = 0; i <= hw_argc - 2; i++)
734 len += strlen(as->argv[i]) + 1;
735 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
736 if (!p) {
737 ti->error = "memory allocation failed";
738 ret = -ENOMEM;
739 goto fail;
740 }
741 j = sprintf(p, "%d", hw_argc - 1);
742 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
743 j = sprintf(p, "%s", as->argv[i]);
744 }
Mike Snitzer498f0102011-08-02 12:32:04 +0100745 dm_consume_args(as, hw_argc - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
747 return 0;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700748fail:
749 kfree(m->hw_handler_name);
750 m->hw_handler_name = NULL;
751 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752}
753
Mike Snitzer498f0102011-08-02 12:32:04 +0100754static int parse_features(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755{
756 int r;
757 unsigned argc;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700758 struct dm_target *ti = m->ti;
Mike Snitzer498f0102011-08-02 12:32:04 +0100759 const char *arg_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
Mike Snitzer498f0102011-08-02 12:32:04 +0100761 static struct dm_arg _args[] = {
Mike Snitzera58a9352012-07-27 15:08:04 +0100762 {0, 6, "invalid number of feature args"},
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100763 {1, 50, "pg_init_retries must be between 1 and 50"},
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000764 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765 };
766
Mike Snitzer498f0102011-08-02 12:32:04 +0100767 r = dm_read_arg_group(_args, as, &argc, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768 if (r)
769 return -EINVAL;
770
771 if (!argc)
772 return 0;
773
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100774 do {
Mike Snitzer498f0102011-08-02 12:32:04 +0100775 arg_name = dm_shift_arg(as);
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100776 argc--;
777
Mike Snitzer498f0102011-08-02 12:32:04 +0100778 if (!strcasecmp(arg_name, "queue_if_no_path")) {
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100779 r = queue_if_no_path(m, 1, 0);
780 continue;
781 }
782
Mike Snitzera58a9352012-07-27 15:08:04 +0100783 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
784 m->retain_attached_hw_handler = 1;
785 continue;
786 }
787
Mike Snitzer498f0102011-08-02 12:32:04 +0100788 if (!strcasecmp(arg_name, "pg_init_retries") &&
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100789 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +0100790 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100791 argc--;
792 continue;
793 }
794
Mike Snitzer498f0102011-08-02 12:32:04 +0100795 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000796 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +0100797 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000798 argc--;
799 continue;
800 }
801
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 ti->error = "Unrecognised multipath feature request";
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100803 r = -EINVAL;
804 } while (argc && !r);
805
806 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807}
808
809static int multipath_ctr(struct dm_target *ti, unsigned int argc,
810 char **argv)
811{
Mike Snitzer498f0102011-08-02 12:32:04 +0100812 /* target arguments */
813 static struct dm_arg _args[] = {
Mike Snitzera490a072011-03-24 13:54:33 +0000814 {0, 1024, "invalid number of priority groups"},
815 {0, 1024, "invalid initial priority group number"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 };
817
818 int r;
819 struct multipath *m;
Mike Snitzer498f0102011-08-02 12:32:04 +0100820 struct dm_arg_set as;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 unsigned pg_count = 0;
822 unsigned next_pg_num;
823
824 as.argc = argc;
825 as.argv = argv;
826
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700827 m = alloc_multipath(ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 if (!m) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700829 ti->error = "can't allocate multipath";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 return -EINVAL;
831 }
832
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700833 r = parse_features(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 if (r)
835 goto bad;
836
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700837 r = parse_hw_handler(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 if (r)
839 goto bad;
840
Mike Snitzer498f0102011-08-02 12:32:04 +0100841 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842 if (r)
843 goto bad;
844
Mike Snitzer498f0102011-08-02 12:32:04 +0100845 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 if (r)
847 goto bad;
848
Mike Snitzera490a072011-03-24 13:54:33 +0000849 if ((!m->nr_priority_groups && next_pg_num) ||
850 (m->nr_priority_groups && !next_pg_num)) {
851 ti->error = "invalid initial priority group";
852 r = -EINVAL;
853 goto bad;
854 }
855
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 /* parse the priority groups */
857 while (as.argc) {
858 struct priority_group *pg;
859
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700860 pg = parse_priority_group(&as, m);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100861 if (IS_ERR(pg)) {
862 r = PTR_ERR(pg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 goto bad;
864 }
865
866 m->nr_valid_paths += pg->nr_pgpaths;
867 list_add_tail(&pg->list, &m->priority_groups);
868 pg_count++;
869 pg->pg_num = pg_count;
870 if (!--next_pg_num)
871 m->next_pg = pg;
872 }
873
874 if (pg_count != m->nr_priority_groups) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700875 ti->error = "priority group count mismatch";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 r = -EINVAL;
877 goto bad;
878 }
879
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +0000880 ti->num_flush_bios = 1;
881 ti->num_discard_bios = 1;
Mike Snitzer042bcef2013-05-10 14:37:16 +0100882 ti->num_write_same_bios = 1;
Mikulas Patocka86279212009-06-22 10:12:24 +0100883
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 return 0;
885
886 bad:
887 free_multipath(m);
888 return r;
889}
890
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000891static void multipath_wait_for_pg_init_completion(struct multipath *m)
892{
893 DECLARE_WAITQUEUE(wait, current);
894 unsigned long flags;
895
896 add_wait_queue(&m->pg_init_wait, &wait);
897
898 while (1) {
899 set_current_state(TASK_UNINTERRUPTIBLE);
900
901 spin_lock_irqsave(&m->lock, flags);
902 if (!m->pg_init_in_progress) {
903 spin_unlock_irqrestore(&m->lock, flags);
904 break;
905 }
906 spin_unlock_irqrestore(&m->lock, flags);
907
908 io_schedule();
909 }
910 set_current_state(TASK_RUNNING);
911
912 remove_wait_queue(&m->pg_init_wait, &wait);
913}
914
915static void flush_multipath_work(struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916{
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000917 unsigned long flags;
918
919 spin_lock_irqsave(&m->lock, flags);
920 m->pg_init_disabled = 1;
921 spin_unlock_irqrestore(&m->lock, flags);
922
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700923 flush_workqueue(kmpath_handlerd);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000924 multipath_wait_for_pg_init_completion(m);
Alasdair G Kergona044d012005-07-12 15:53:02 -0700925 flush_workqueue(kmultipathd);
Tejun Heo43829732012-08-20 14:51:24 -0700926 flush_work(&m->trigger_event);
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000927
928 spin_lock_irqsave(&m->lock, flags);
929 m->pg_init_disabled = 0;
930 spin_unlock_irqrestore(&m->lock, flags);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +0000931}
932
933static void multipath_dtr(struct dm_target *ti)
934{
935 struct multipath *m = ti->private;
936
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000937 flush_multipath_work(m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938 free_multipath(m);
939}
940
941/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 * Take a path out of use.
943 */
944static int fail_path(struct pgpath *pgpath)
945{
946 unsigned long flags;
947 struct multipath *m = pgpath->pg->m;
948
949 spin_lock_irqsave(&m->lock, flags);
950
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100951 if (!pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 goto out;
953
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700954 DMWARN("Failing path %s.", pgpath->path.dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955
956 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100957 pgpath->is_active = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 pgpath->fail_count++;
959
960 m->nr_valid_paths--;
961
962 if (pgpath == m->current_pgpath)
963 m->current_pgpath = NULL;
964
Mike Andersonb15546f2007-10-19 22:48:02 +0100965 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
966 pgpath->path.dev->name, m->nr_valid_paths);
967
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +0000968 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969
970out:
971 spin_unlock_irqrestore(&m->lock, flags);
972
973 return 0;
974}
975
976/*
977 * Reinstate a previously-failed path
978 */
979static int reinstate_path(struct pgpath *pgpath)
980{
Hannes Reinecke63d832c2014-05-26 14:45:39 +0200981 int r = 0, run_queue = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 unsigned long flags;
983 struct multipath *m = pgpath->pg->m;
984
985 spin_lock_irqsave(&m->lock, flags);
986
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100987 if (pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 goto out;
989
Alasdair G Kergondef052d2008-07-21 12:00:31 +0100990 if (!pgpath->pg->ps.type->reinstate_path) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 DMWARN("Reinstate path not supported by path selector %s",
992 pgpath->pg->ps.type->name);
993 r = -EINVAL;
994 goto out;
995 }
996
997 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
998 if (r)
999 goto out;
1000
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001001 pgpath->is_active = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002
Hannes Reineckee8099172014-02-28 15:33:44 +01001003 if (!m->nr_valid_paths++) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001004 m->current_pgpath = NULL;
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001005 run_queue = 1;
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001006 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001007 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001008 m->pg_init_in_progress++;
1009 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010
Mike Andersonb15546f2007-10-19 22:48:02 +01001011 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1012 pgpath->path.dev->name, m->nr_valid_paths);
1013
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001014 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
1016out:
1017 spin_unlock_irqrestore(&m->lock, flags);
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001018 if (run_queue)
1019 dm_table_run_md_queue_async(m->ti->table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020
1021 return r;
1022}
1023
1024/*
1025 * Fail or reinstate all paths that match the provided struct dm_dev.
1026 */
1027static int action_dev(struct multipath *m, struct dm_dev *dev,
1028 action_fn action)
1029{
Mike Snitzer19040c02011-03-24 13:54:31 +00001030 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 struct pgpath *pgpath;
1032 struct priority_group *pg;
1033
1034 list_for_each_entry(pg, &m->priority_groups, list) {
1035 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1036 if (pgpath->path.dev == dev)
1037 r = action(pgpath);
1038 }
1039 }
1040
1041 return r;
1042}
1043
1044/*
1045 * Temporarily try to avoid having to use the specified PG
1046 */
1047static void bypass_pg(struct multipath *m, struct priority_group *pg,
1048 int bypassed)
1049{
1050 unsigned long flags;
1051
1052 spin_lock_irqsave(&m->lock, flags);
1053
1054 pg->bypassed = bypassed;
1055 m->current_pgpath = NULL;
1056 m->current_pg = NULL;
1057
1058 spin_unlock_irqrestore(&m->lock, flags);
1059
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001060 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061}
1062
1063/*
1064 * Switch to using the specified PG from the next I/O that gets mapped
1065 */
1066static int switch_pg_num(struct multipath *m, const char *pgstr)
1067{
1068 struct priority_group *pg;
1069 unsigned pgnum;
1070 unsigned long flags;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001071 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001073 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 (pgnum > m->nr_priority_groups)) {
1075 DMWARN("invalid PG number supplied to switch_pg_num");
1076 return -EINVAL;
1077 }
1078
1079 spin_lock_irqsave(&m->lock, flags);
1080 list_for_each_entry(pg, &m->priority_groups, list) {
1081 pg->bypassed = 0;
1082 if (--pgnum)
1083 continue;
1084
1085 m->current_pgpath = NULL;
1086 m->current_pg = NULL;
1087 m->next_pg = pg;
1088 }
1089 spin_unlock_irqrestore(&m->lock, flags);
1090
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001091 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 return 0;
1093}
1094
1095/*
1096 * Set/clear bypassed status of a PG.
1097 * PGs are numbered upwards from 1 in the order they were declared.
1098 */
1099static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
1100{
1101 struct priority_group *pg;
1102 unsigned pgnum;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001103 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001105 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106 (pgnum > m->nr_priority_groups)) {
1107 DMWARN("invalid PG number supplied to bypass_pg");
1108 return -EINVAL;
1109 }
1110
1111 list_for_each_entry(pg, &m->priority_groups, list) {
1112 if (!--pgnum)
1113 break;
1114 }
1115
1116 bypass_pg(m, pg, bypassed);
1117 return 0;
1118}
1119
1120/*
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001121 * Should we retry pg_init immediately?
1122 */
1123static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1124{
1125 unsigned long flags;
1126 int limit_reached = 0;
1127
1128 spin_lock_irqsave(&m->lock, flags);
1129
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +00001130 if (m->pg_init_count <= m->pg_init_retries && !m->pg_init_disabled)
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001131 m->pg_init_required = 1;
1132 else
1133 limit_reached = 1;
1134
1135 spin_unlock_irqrestore(&m->lock, flags);
1136
1137 return limit_reached;
1138}
1139
Chandra Seetharaman3ae31f62009-10-21 09:22:46 -07001140static void pg_init_done(void *data, int errors)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001141{
Moger, Babu83c0d5d2010-03-06 02:29:45 +00001142 struct pgpath *pgpath = data;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001143 struct priority_group *pg = pgpath->pg;
1144 struct multipath *m = pg->m;
1145 unsigned long flags;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001146 unsigned delay_retry = 0;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001147
1148 /* device or driver problems */
1149 switch (errors) {
1150 case SCSI_DH_OK:
1151 break;
1152 case SCSI_DH_NOSYS:
1153 if (!m->hw_handler_name) {
1154 errors = 0;
1155 break;
1156 }
Moger, Babuf7b934c2010-03-06 02:29:49 +00001157 DMERR("Could not failover the device: Handler scsi_dh_%s "
1158 "Error %d.", m->hw_handler_name, errors);
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001159 /*
1160 * Fail path for now, so we do not ping pong
1161 */
1162 fail_path(pgpath);
1163 break;
1164 case SCSI_DH_DEV_TEMP_BUSY:
1165 /*
1166 * Probably doing something like FW upgrade on the
1167 * controller so try the other pg.
1168 */
1169 bypass_pg(m, pg, 1);
1170 break;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001171 case SCSI_DH_RETRY:
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001172 /* Wait before retrying. */
1173 delay_retry = 1;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001174 case SCSI_DH_IMM_RETRY:
1175 case SCSI_DH_RES_TEMP_UNAVAIL:
1176 if (pg_init_limit_reached(m, pgpath))
1177 fail_path(pgpath);
1178 errors = 0;
1179 break;
1180 default:
1181 /*
1182 * We probably do not want to fail the path for a device
1183 * error, but this is what the old dm did. In future
1184 * patches we can do more advanced handling.
1185 */
1186 fail_path(pgpath);
1187 }
1188
1189 spin_lock_irqsave(&m->lock, flags);
1190 if (errors) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001191 if (pgpath == m->current_pgpath) {
1192 DMERR("Could not failover device. Error %d.", errors);
1193 m->current_pgpath = NULL;
1194 m->current_pg = NULL;
1195 }
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001196 } else if (!m->pg_init_required)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001197 pg->bypassed = 0;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001198
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001199 if (--m->pg_init_in_progress)
1200 /* Activations of other paths are still on going */
1201 goto out;
1202
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001203 if (m->pg_init_required) {
1204 m->pg_init_delay_retry = delay_retry;
1205 if (__pg_init_all_paths(m))
1206 goto out;
1207 }
1208 m->queue_io = 0;
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001209
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001210 /*
1211 * Wake up any thread waiting to suspend.
1212 */
1213 wake_up(&m->pg_init_wait);
1214
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001215out:
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001216 spin_unlock_irqrestore(&m->lock, flags);
1217}
1218
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001219static void activate_path(struct work_struct *work)
1220{
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001221 struct pgpath *pgpath =
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001222 container_of(work, struct pgpath, activate_path.work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001223
Hannes Reinecke3a017502014-02-28 15:33:49 +01001224 if (pgpath->is_active)
1225 scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
1226 pg_init_done, pgpath);
1227 else
1228 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001229}
1230
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001231static int noretry_error(int error)
1232{
1233 switch (error) {
1234 case -EOPNOTSUPP:
1235 case -EREMOTEIO:
1236 case -EILSEQ:
1237 case -ENODATA:
Jun'ichi Nomuracc9d3c32013-09-13 14:54:30 +09001238 case -ENOSPC:
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001239 return 1;
1240 }
1241
1242 /* Anything else could be a path failure, so should be retried */
1243 return 0;
1244}
1245
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246/*
1247 * end_io handling
1248 */
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001249static int do_end_io(struct multipath *m, struct request *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001250 int error, struct dm_mpath_io *mpio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251{
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001252 /*
1253 * We don't queue any clone request inside the multipath target
1254 * during end I/O handling, since those clone requests don't have
1255 * bio clones. If we queue them inside the multipath target,
1256 * we need to make bio clones, that requires memory allocation.
1257 * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
1258 * don't have bio clones.)
1259 * Instead of queueing the clone request here, we queue the original
1260 * request into dm core, which will remake a clone request and
1261 * clone bios for it and resubmit it later.
1262 */
1263 int r = DM_ENDIO_REQUEUE;
Stefan Bader640eb3b2005-11-21 21:32:35 -08001264 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001266 if (!error && !clone->errors)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267 return 0; /* I/O complete */
1268
Mike Snitzer7eee4ae2014-06-02 15:50:06 -04001269 if (noretry_error(error))
Mike Snitzer959eb4e2010-08-12 04:14:32 +01001270 return error;
1271
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001272 if (mpio->pgpath)
1273 fail_path(mpio->pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274
Stefan Bader640eb3b2005-11-21 21:32:35 -08001275 spin_lock_irqsave(&m->lock, flags);
Hannes Reinecke751b2a72011-01-18 10:13:12 +01001276 if (!m->nr_valid_paths) {
1277 if (!m->queue_if_no_path) {
1278 if (!__must_push_back(m))
1279 r = -EIO;
1280 } else {
1281 if (error == -EBADE)
1282 r = error;
1283 }
1284 }
Stefan Bader640eb3b2005-11-21 21:32:35 -08001285 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001287 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288}
1289
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001290static int multipath_end_io(struct dm_target *ti, struct request *clone,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 int error, union map_info *map_context)
1292{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001293 struct multipath *m = ti->private;
1294 struct dm_mpath_io *mpio = map_context->ptr;
Wei Yongjuna71a2612012-10-12 16:59:42 +01001295 struct pgpath *pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296 struct path_selector *ps;
1297 int r;
1298
Jun'ichi Nomura466891f2012-03-28 18:41:25 +01001299 BUG_ON(!mpio);
1300
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001301 r = do_end_io(m, clone, error, mpio);
Wei Yongjuna71a2612012-10-12 16:59:42 +01001302 pgpath = mpio->pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303 if (pgpath) {
1304 ps = &pgpath->pg->ps;
1305 if (ps->type->end_io)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001306 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307 }
Jun'ichi Nomura466891f2012-03-28 18:41:25 +01001308 clear_mapinfo(m, map_context);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309
1310 return r;
1311}
1312
1313/*
1314 * Suspend can't complete until all the I/O is processed so if
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001315 * the last path fails we must error any remaining I/O.
1316 * Note that if the freeze_bdev fails while suspending, the
1317 * queue_if_no_path state is lost - userspace should reset it.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318 */
1319static void multipath_presuspend(struct dm_target *ti)
1320{
1321 struct multipath *m = (struct multipath *) ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322
Alasdair G Kergon485ef692005-09-27 21:45:45 -07001323 queue_if_no_path(m, 0, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324}
1325
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001326static void multipath_postsuspend(struct dm_target *ti)
1327{
Mike Anderson6380f262009-12-10 23:52:21 +00001328 struct multipath *m = ti->private;
1329
1330 mutex_lock(&m->work_mutex);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001331 flush_multipath_work(m);
Mike Anderson6380f262009-12-10 23:52:21 +00001332 mutex_unlock(&m->work_mutex);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001333}
1334
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001335/*
1336 * Restore the queue_if_no_path setting.
1337 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338static void multipath_resume(struct dm_target *ti)
1339{
1340 struct multipath *m = (struct multipath *) ti->private;
1341 unsigned long flags;
1342
1343 spin_lock_irqsave(&m->lock, flags);
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001344 m->queue_if_no_path = m->saved_queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345 spin_unlock_irqrestore(&m->lock, flags);
1346}
1347
1348/*
1349 * Info output has the following format:
1350 * num_multipath_feature_args [multipath_feature_args]*
1351 * num_handler_status_args [handler_status_args]*
1352 * num_groups init_group_number
1353 * [A|D|E num_ps_status_args [ps_status_args]*
1354 * num_paths num_selector_args
1355 * [path_dev A|F fail_count [selector_args]* ]+ ]+
1356 *
1357 * Table output has the following format (identical to the constructor string):
1358 * num_feature_args [features_args]*
1359 * num_handler_args hw_handler [hw_handler_args]*
1360 * num_groups init_group_number
1361 * [priority selector-name num_ps_args [ps_args]*
1362 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1363 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00001364static void multipath_status(struct dm_target *ti, status_type_t type,
1365 unsigned status_flags, char *result, unsigned maxlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366{
1367 int sz = 0;
1368 unsigned long flags;
1369 struct multipath *m = (struct multipath *) ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370 struct priority_group *pg;
1371 struct pgpath *p;
1372 unsigned pg_num;
1373 char state;
1374
1375 spin_lock_irqsave(&m->lock, flags);
1376
1377 /* Features */
1378 if (type == STATUSTYPE_INFO)
Hannes Reineckee8099172014-02-28 15:33:44 +01001379 DMEMIT("2 %u %u ", m->queue_io, m->pg_init_count);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001380 else {
1381 DMEMIT("%u ", m->queue_if_no_path +
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001382 (m->pg_init_retries > 0) * 2 +
Mike Snitzera58a9352012-07-27 15:08:04 +01001383 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1384 m->retain_attached_hw_handler);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001385 if (m->queue_if_no_path)
1386 DMEMIT("queue_if_no_path ");
1387 if (m->pg_init_retries)
1388 DMEMIT("pg_init_retries %u ", m->pg_init_retries);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001389 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1390 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
Mike Snitzera58a9352012-07-27 15:08:04 +01001391 if (m->retain_attached_hw_handler)
1392 DMEMIT("retain_attached_hw_handler ");
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001393 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001395 if (!m->hw_handler_name || type == STATUSTYPE_INFO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 DMEMIT("0 ");
1397 else
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001398 DMEMIT("1 %s ", m->hw_handler_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399
1400 DMEMIT("%u ", m->nr_priority_groups);
1401
1402 if (m->next_pg)
1403 pg_num = m->next_pg->pg_num;
1404 else if (m->current_pg)
1405 pg_num = m->current_pg->pg_num;
1406 else
Mike Snitzera490a072011-03-24 13:54:33 +00001407 pg_num = (m->nr_priority_groups ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408
1409 DMEMIT("%u ", pg_num);
1410
1411 switch (type) {
1412 case STATUSTYPE_INFO:
1413 list_for_each_entry(pg, &m->priority_groups, list) {
1414 if (pg->bypassed)
1415 state = 'D'; /* Disabled */
1416 else if (pg == m->current_pg)
1417 state = 'A'; /* Currently Active */
1418 else
1419 state = 'E'; /* Enabled */
1420
1421 DMEMIT("%c ", state);
1422
1423 if (pg->ps.type->status)
1424 sz += pg->ps.type->status(&pg->ps, NULL, type,
1425 result + sz,
1426 maxlen - sz);
1427 else
1428 DMEMIT("0 ");
1429
1430 DMEMIT("%u %u ", pg->nr_pgpaths,
1431 pg->ps.type->info_args);
1432
1433 list_for_each_entry(p, &pg->pgpaths, list) {
1434 DMEMIT("%s %s %u ", p->path.dev->name,
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001435 p->is_active ? "A" : "F",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 p->fail_count);
1437 if (pg->ps.type->status)
1438 sz += pg->ps.type->status(&pg->ps,
1439 &p->path, type, result + sz,
1440 maxlen - sz);
1441 }
1442 }
1443 break;
1444
1445 case STATUSTYPE_TABLE:
1446 list_for_each_entry(pg, &m->priority_groups, list) {
1447 DMEMIT("%s ", pg->ps.type->name);
1448
1449 if (pg->ps.type->status)
1450 sz += pg->ps.type->status(&pg->ps, NULL, type,
1451 result + sz,
1452 maxlen - sz);
1453 else
1454 DMEMIT("0 ");
1455
1456 DMEMIT("%u %u ", pg->nr_pgpaths,
1457 pg->ps.type->table_args);
1458
1459 list_for_each_entry(p, &pg->pgpaths, list) {
1460 DMEMIT("%s ", p->path.dev->name);
1461 if (pg->ps.type->status)
1462 sz += pg->ps.type->status(&pg->ps,
1463 &p->path, type, result + sz,
1464 maxlen - sz);
1465 }
1466 }
1467 break;
1468 }
1469
1470 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471}
1472
1473static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1474{
Mike Anderson6380f262009-12-10 23:52:21 +00001475 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476 struct dm_dev *dev;
1477 struct multipath *m = (struct multipath *) ti->private;
1478 action_fn action;
1479
Mike Anderson6380f262009-12-10 23:52:21 +00001480 mutex_lock(&m->work_mutex);
1481
Kiyoshi Uedac2f3d242009-12-10 23:52:27 +00001482 if (dm_suspended(ti)) {
1483 r = -EBUSY;
1484 goto out;
1485 }
1486
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 if (argc == 1) {
Mike Snitzer498f0102011-08-02 12:32:04 +01001488 if (!strcasecmp(argv[0], "queue_if_no_path")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001489 r = queue_if_no_path(m, 1, 0);
1490 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001491 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001492 r = queue_if_no_path(m, 0, 0);
1493 goto out;
1494 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495 }
1496
Mike Anderson6380f262009-12-10 23:52:21 +00001497 if (argc != 2) {
Jose Castilloa356e422014-01-29 17:52:45 +01001498 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
Mike Anderson6380f262009-12-10 23:52:21 +00001499 goto out;
1500 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501
Mike Snitzer498f0102011-08-02 12:32:04 +01001502 if (!strcasecmp(argv[0], "disable_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001503 r = bypass_pg_num(m, argv[1], 1);
1504 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001505 } else if (!strcasecmp(argv[0], "enable_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001506 r = bypass_pg_num(m, argv[1], 0);
1507 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001508 } else if (!strcasecmp(argv[0], "switch_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001509 r = switch_pg_num(m, argv[1]);
1510 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001511 } else if (!strcasecmp(argv[0], "reinstate_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512 action = reinstate_path;
Mike Snitzer498f0102011-08-02 12:32:04 +01001513 else if (!strcasecmp(argv[0], "fail_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 action = fail_path;
Mike Anderson6380f262009-12-10 23:52:21 +00001515 else {
Jose Castilloa356e422014-01-29 17:52:45 +01001516 DMWARN("Unrecognised multipath message received: %s", argv[0]);
Mike Anderson6380f262009-12-10 23:52:21 +00001517 goto out;
1518 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +00001520 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001522 DMWARN("message: error getting device %s",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523 argv[1]);
Mike Anderson6380f262009-12-10 23:52:21 +00001524 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525 }
1526
1527 r = action_dev(m, dev, action);
1528
1529 dm_put_device(ti, dev);
1530
Mike Anderson6380f262009-12-10 23:52:21 +00001531out:
1532 mutex_unlock(&m->work_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001533 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534}
1535
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001536static int multipath_prepare_ioctl(struct dm_target *ti,
1537 struct block_device **bdev, fmode_t *mode)
Milan Broz9af4aa32006-10-03 01:15:20 -07001538{
Mikulas Patocka35991652012-06-03 00:29:58 +01001539 struct multipath *m = ti->private;
Milan Broz9af4aa32006-10-03 01:15:20 -07001540 unsigned long flags;
Mikulas Patocka35991652012-06-03 00:29:58 +01001541 int r;
1542
Milan Broz9af4aa32006-10-03 01:15:20 -07001543 spin_lock_irqsave(&m->lock, flags);
1544
1545 if (!m->current_pgpath)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001546 __choose_pgpath(m, 0);
Milan Broz9af4aa32006-10-03 01:15:20 -07001547
Junichi Nomura43e43c92015-11-17 09:36:56 +00001548 if (m->current_pgpath) {
1549 if (!m->queue_io) {
1550 *bdev = m->current_pgpath->path.dev->bdev;
1551 *mode = m->current_pgpath->path.dev->mode;
1552 r = 0;
1553 } else {
1554 /* pg_init has not started or completed */
1555 r = -ENOTCONN;
1556 }
1557 } else {
1558 /* No path is available */
1559 if (m->queue_if_no_path)
1560 r = -ENOTCONN;
1561 else
1562 r = -EIO;
Milan Broze90dae12006-10-03 01:15:22 -07001563 }
Milan Broz9af4aa32006-10-03 01:15:20 -07001564
Milan Broz9af4aa32006-10-03 01:15:20 -07001565 spin_unlock_irqrestore(&m->lock, flags);
1566
Junichi Nomura5bbbfdf2015-11-17 09:39:26 +00001567 if (r == -ENOTCONN) {
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001568 spin_lock_irqsave(&m->lock, flags);
1569 if (!m->current_pg) {
1570 /* Path status changed, redo selection */
1571 __choose_pgpath(m, 0);
1572 }
1573 if (m->pg_init_required)
1574 __pg_init_all_paths(m);
Mike Snitzer4cdd2ad2014-05-13 13:49:39 -04001575 spin_unlock_irqrestore(&m->lock, flags);
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001576 dm_table_run_md_queue_async(m->ti->table);
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001577 }
Mikulas Patocka35991652012-06-03 00:29:58 +01001578
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001579 /*
1580 * Only pass ioctls through if the device sizes match exactly.
1581 */
1582 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1583 return 1;
1584 return r;
Milan Broz9af4aa32006-10-03 01:15:20 -07001585}
1586
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001587static int multipath_iterate_devices(struct dm_target *ti,
1588 iterate_devices_callout_fn fn, void *data)
1589{
1590 struct multipath *m = ti->private;
1591 struct priority_group *pg;
1592 struct pgpath *p;
1593 int ret = 0;
1594
1595 list_for_each_entry(pg, &m->priority_groups, list) {
1596 list_for_each_entry(p, &pg->pgpaths, list) {
Mike Snitzer5dea2712009-07-23 20:30:42 +01001597 ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001598 if (ret)
1599 goto out;
1600 }
1601 }
1602
1603out:
1604 return ret;
1605}
1606
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001607static int __pgpath_busy(struct pgpath *pgpath)
1608{
1609 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1610
Mike Snitzer52b09912015-02-23 16:36:41 -05001611 return blk_lld_busy(q);
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001612}
1613
1614/*
1615 * We return "busy", only when we can map I/Os but underlying devices
1616 * are busy (so even if we map I/Os now, the I/Os will wait on
1617 * the underlying queue).
1618 * In other words, if we want to kill I/Os or queue them inside us
1619 * due to map unavailability, we don't return "busy". Otherwise,
1620 * dm core won't give us the I/Os and we can't do what we want.
1621 */
1622static int multipath_busy(struct dm_target *ti)
1623{
1624 int busy = 0, has_active = 0;
1625 struct multipath *m = ti->private;
1626 struct priority_group *pg;
1627 struct pgpath *pgpath;
1628 unsigned long flags;
1629
1630 spin_lock_irqsave(&m->lock, flags);
1631
Jun'ichi Nomura7a7a3b42014-07-08 00:55:14 +00001632 /* pg_init in progress or no paths available */
1633 if (m->pg_init_in_progress ||
1634 (!m->nr_valid_paths && m->queue_if_no_path)) {
Hannes Reineckeb63349a2013-10-01 11:49:56 +02001635 busy = 1;
1636 goto out;
1637 }
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001638 /* Guess which priority_group will be used at next mapping time */
1639 if (unlikely(!m->current_pgpath && m->next_pg))
1640 pg = m->next_pg;
1641 else if (likely(m->current_pg))
1642 pg = m->current_pg;
1643 else
1644 /*
1645 * We don't know which pg will be used at next mapping time.
1646 * We don't call __choose_pgpath() here to avoid to trigger
1647 * pg_init just by busy checking.
1648 * So we don't know whether underlying devices we will be using
1649 * at next mapping time are busy or not. Just try mapping.
1650 */
1651 goto out;
1652
1653 /*
1654 * If there is one non-busy active path at least, the path selector
1655 * will be able to select it. So we consider such a pg as not busy.
1656 */
1657 busy = 1;
1658 list_for_each_entry(pgpath, &pg->pgpaths, list)
1659 if (pgpath->is_active) {
1660 has_active = 1;
1661
1662 if (!__pgpath_busy(pgpath)) {
1663 busy = 0;
1664 break;
1665 }
1666 }
1667
1668 if (!has_active)
1669 /*
1670 * No active path in this pg, so this pg won't be used and
1671 * the current_pg will be changed at next mapping time.
1672 * We need to try mapping to determine it.
1673 */
1674 busy = 0;
1675
1676out:
1677 spin_unlock_irqrestore(&m->lock, flags);
1678
1679 return busy;
1680}
1681
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682/*-----------------------------------------------------------------
1683 * Module setup
1684 *---------------------------------------------------------------*/
1685static struct target_type multipath_target = {
1686 .name = "multipath",
Mike Snitzer16f12262016-01-31 17:22:27 -05001687 .version = {1, 11, 0},
1688 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689 .module = THIS_MODULE,
1690 .ctr = multipath_ctr,
1691 .dtr = multipath_dtr,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001692 .map_rq = multipath_map,
Mike Snitzere5863d92014-12-17 21:08:12 -05001693 .clone_and_map_rq = multipath_clone_and_map,
1694 .release_clone_rq = multipath_release_clone,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001695 .rq_end_io = multipath_end_io,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001696 .presuspend = multipath_presuspend,
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001697 .postsuspend = multipath_postsuspend,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698 .resume = multipath_resume,
1699 .status = multipath_status,
1700 .message = multipath_message,
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001701 .prepare_ioctl = multipath_prepare_ioctl,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001702 .iterate_devices = multipath_iterate_devices,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001703 .busy = multipath_busy,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704};
1705
1706static int __init dm_multipath_init(void)
1707{
1708 int r;
1709
1710 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001711 _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001712 if (!_mpio_cache)
1713 return -ENOMEM;
1714
1715 r = dm_register_target(&multipath_target);
1716 if (r < 0) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001717 DMERR("register failed %d", r);
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001718 r = -EINVAL;
1719 goto bad_register_target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001720 }
1721
Tejun Heo4d4d66a2011-01-13 19:59:57 +00001722 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001723 if (!kmultipathd) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001724 DMERR("failed to create workqueue kmpathd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001725 r = -ENOMEM;
1726 goto bad_alloc_kmultipathd;
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001727 }
1728
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001729 /*
1730 * A separate workqueue is used to handle the device handlers
1731 * to avoid overloading existing workqueue. Overloading the
1732 * old workqueue would also create a bottleneck in the
1733 * path of the storage hardware device activation.
1734 */
Tejun Heo4d4d66a2011-01-13 19:59:57 +00001735 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
1736 WQ_MEM_RECLAIM);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001737 if (!kmpath_handlerd) {
1738 DMERR("failed to create workqueue kmpath_handlerd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001739 r = -ENOMEM;
1740 goto bad_alloc_kmpath_handlerd;
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001741 }
1742
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001743 DMINFO("version %u.%u.%u loaded",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744 multipath_target.version[0], multipath_target.version[1],
1745 multipath_target.version[2]);
1746
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001747 return 0;
1748
1749bad_alloc_kmpath_handlerd:
1750 destroy_workqueue(kmultipathd);
1751bad_alloc_kmultipathd:
1752 dm_unregister_target(&multipath_target);
1753bad_register_target:
1754 kmem_cache_destroy(_mpio_cache);
1755
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756 return r;
1757}
1758
1759static void __exit dm_multipath_exit(void)
1760{
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001761 destroy_workqueue(kmpath_handlerd);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001762 destroy_workqueue(kmultipathd);
1763
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001764 dm_unregister_target(&multipath_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765 kmem_cache_destroy(_mpio_cache);
1766}
1767
Linus Torvalds1da177e2005-04-16 15:20:36 -07001768module_init(dm_multipath_init);
1769module_exit(dm_multipath_exit);
1770
1771MODULE_DESCRIPTION(DM_NAME " multipath target");
1772MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
1773MODULE_LICENSE("GPL");