blob: 63953477a07c36e771a32d5bde686bd0f05890f1 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
Mikulas Patocka586e80e2008-10-21 17:44:59 +01008#include <linux/device-mapper.h>
9
Mike Snitzerf4790822013-09-12 18:06:12 -040010#include "dm.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include "dm-path-selector.h"
Mike Andersonb15546f2007-10-19 22:48:02 +010012#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070013
Mike Snitzere5863d92014-12-17 21:08:12 -050014#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/ctype.h>
16#include <linux/init.h>
17#include <linux/mempool.h>
18#include <linux/module.h>
19#include <linux/pagemap.h>
20#include <linux/slab.h>
21#include <linux/time.h>
22#include <linux/workqueue.h>
Mikulas Patocka35991652012-06-03 00:29:58 +010023#include <linux/delay.h>
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070024#include <scsi/scsi_dh.h>
Arun Sharma600634972011-07-26 16:09:06 -070025#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
Alasdair G Kergon72d94862006-06-26 00:27:35 -070027#define DM_MSG_PREFIX "multipath"
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000028#define DM_PG_INIT_DELAY_MSECS 2000
29#define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31/* Path properties */
32struct pgpath {
33 struct list_head list;
34
35 struct priority_group *pg; /* Owning PG */
Kiyoshi Ueda66800732008-10-10 13:36:58 +010036 unsigned is_active; /* Path status */
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 unsigned fail_count; /* Cumulative failure count */
38
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -080039 struct dm_path path;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000040 struct delayed_work activate_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -070041};
42
43#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
44
45/*
46 * Paths are grouped into Priority Groups and numbered from 1 upwards.
47 * Each has a path selector which controls which path gets used.
48 */
49struct priority_group {
50 struct list_head list;
51
52 struct multipath *m; /* Owning multipath instance */
53 struct path_selector ps;
54
55 unsigned pg_num; /* Reference number */
56 unsigned bypassed; /* Temporarily bypass this PG? */
57
58 unsigned nr_pgpaths; /* Number of paths in PG */
59 struct list_head pgpaths;
60};
61
62/* Multipath context */
63struct multipath {
64 struct list_head list;
65 struct dm_target *ti;
66
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070067 const char *hw_handler_name;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -070068 char *hw_handler_params;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000069
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010070 spinlock_t lock;
71
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 unsigned nr_priority_groups;
73 struct list_head priority_groups;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000074
75 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
76
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 unsigned pg_init_required; /* pg_init needs calling? */
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -070078 unsigned pg_init_in_progress; /* Only one pg_init allowed at once */
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000079 unsigned pg_init_delay_retry; /* Delay pg_init retry? */
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
81 unsigned nr_valid_paths; /* Total number of usable paths */
82 struct pgpath *current_pgpath;
83 struct priority_group *current_pg;
84 struct priority_group *next_pg; /* Switch to this PG if set */
85 unsigned repeat_count; /* I/Os left before calling PS again */
86
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010087 unsigned queue_io:1; /* Must we queue all I/O? */
88 unsigned queue_if_no_path:1; /* Queue I/O if last path fails? */
89 unsigned saved_queue_if_no_path:1; /* Saved state during suspension */
Mike Snitzera58a9352012-07-27 15:08:04 +010090 unsigned retain_attached_hw_handler:1; /* If there's already a hw_handler present, don't change it. */
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +000091 unsigned pg_init_disabled:1; /* pg_init is not currently allowed */
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010092
Dave Wysochanskic9e45582007-10-19 22:47:53 +010093 unsigned pg_init_retries; /* Number of times to retry pg_init */
94 unsigned pg_init_count; /* Number of times pg_init called */
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000095 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 struct work_struct trigger_event;
98
99 /*
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100100 * We must use a mempool of dm_mpath_io structs so that we
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 * can resubmit bios on error.
102 */
103 mempool_t *mpio_pool;
Mike Anderson6380f262009-12-10 23:52:21 +0000104
105 struct mutex work_mutex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106};
107
108/*
109 * Context information attached to each bio we process.
110 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100111struct dm_mpath_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 struct pgpath *pgpath;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100113 size_t nr_bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114};
115
116typedef int (*action_fn) (struct pgpath *pgpath);
117
Christoph Lametere18b8902006-12-06 20:33:20 -0800118static struct kmem_cache *_mpio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700120static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
David Howellsc4028952006-11-22 14:57:56 +0000121static void trigger_event(struct work_struct *work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700122static void activate_path(struct work_struct *work);
Hannes Reineckee8099172014-02-28 15:33:44 +0100123static int __pgpath_busy(struct pgpath *pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124
125
126/*-----------------------------------------------
127 * Allocation routines
128 *-----------------------------------------------*/
129
130static struct pgpath *alloc_pgpath(void)
131{
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700132 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
Mike Anderson224cb3e2008-08-29 09:36:09 +0200134 if (pgpath) {
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100135 pgpath->is_active = 1;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000136 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
Mike Anderson224cb3e2008-08-29 09:36:09 +0200137 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
139 return pgpath;
140}
141
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100142static void free_pgpath(struct pgpath *pgpath)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143{
144 kfree(pgpath);
145}
146
147static struct priority_group *alloc_priority_group(void)
148{
149 struct priority_group *pg;
150
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700151 pg = kzalloc(sizeof(*pg), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700153 if (pg)
154 INIT_LIST_HEAD(&pg->pgpaths);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155
156 return pg;
157}
158
159static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
160{
161 struct pgpath *pgpath, *tmp;
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700162 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
164 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
165 list_del(&pgpath->list);
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700166 if (m->hw_handler_name)
167 scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 dm_put_device(ti, pgpath->path.dev);
169 free_pgpath(pgpath);
170 }
171}
172
173static void free_priority_group(struct priority_group *pg,
174 struct dm_target *ti)
175{
176 struct path_selector *ps = &pg->ps;
177
178 if (ps->type) {
179 ps->type->destroy(ps);
180 dm_put_path_selector(ps->type);
181 }
182
183 free_pgpaths(&pg->pgpaths, ti);
184 kfree(pg);
185}
186
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700187static struct multipath *alloc_multipath(struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188{
189 struct multipath *m;
Mike Snitzerf4790822013-09-12 18:06:12 -0400190 unsigned min_ios = dm_get_reserved_rq_based_ios();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700192 m = kzalloc(sizeof(*m), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 if (m) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 INIT_LIST_HEAD(&m->priority_groups);
195 spin_lock_init(&m->lock);
196 m->queue_io = 1;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000197 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
David Howellsc4028952006-11-22 14:57:56 +0000198 INIT_WORK(&m->trigger_event, trigger_event);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000199 init_waitqueue_head(&m->pg_init_wait);
Mike Anderson6380f262009-12-10 23:52:21 +0000200 mutex_init(&m->work_mutex);
Mike Snitzerf4790822013-09-12 18:06:12 -0400201 m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 if (!m->mpio_pool) {
203 kfree(m);
204 return NULL;
205 }
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700206 m->ti = ti;
207 ti->private = m;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 }
209
210 return m;
211}
212
213static void free_multipath(struct multipath *m)
214{
215 struct priority_group *pg, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
218 list_del(&pg->list);
219 free_priority_group(pg, m->ti);
220 }
221
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700222 kfree(m->hw_handler_name);
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700223 kfree(m->hw_handler_params);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 mempool_destroy(m->mpio_pool);
225 kfree(m);
226}
227
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100228static int set_mapinfo(struct multipath *m, union map_info *info)
229{
230 struct dm_mpath_io *mpio;
231
232 mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
233 if (!mpio)
234 return -ENOMEM;
235
236 memset(mpio, 0, sizeof(*mpio));
237 info->ptr = mpio;
238
239 return 0;
240}
241
242static void clear_mapinfo(struct multipath *m, union map_info *info)
243{
244 struct dm_mpath_io *mpio = info->ptr;
245
246 info->ptr = NULL;
247 mempool_free(mpio, m->mpio_pool);
248}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
250/*-----------------------------------------------
251 * Path selection
252 *-----------------------------------------------*/
253
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100254static int __pg_init_all_paths(struct multipath *m)
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000255{
256 struct pgpath *pgpath;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000257 unsigned long pg_init_delay = 0;
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000258
Hannes Reinecke17f4ff42014-02-28 15:33:42 +0100259 if (m->pg_init_in_progress || m->pg_init_disabled)
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100260 return 0;
Hannes Reinecke17f4ff42014-02-28 15:33:42 +0100261
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000262 m->pg_init_count++;
263 m->pg_init_required = 0;
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100264
265 /* Check here to reset pg_init_required */
266 if (!m->current_pg)
267 return 0;
268
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000269 if (m->pg_init_delay_retry)
270 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
271 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000272 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
273 /* Skip failed paths */
274 if (!pgpath->is_active)
275 continue;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000276 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
277 pg_init_delay))
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000278 m->pg_init_in_progress++;
279 }
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100280 return m->pg_init_in_progress;
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000281}
282
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
284{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 m->current_pg = pgpath->pg;
286
287 /* Must we initialise the PG first, and queue I/O till it's ready? */
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700288 if (m->hw_handler_name) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 m->pg_init_required = 1;
290 m->queue_io = 1;
291 } else {
292 m->pg_init_required = 0;
293 m->queue_io = 0;
294 }
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100295
296 m->pg_init_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297}
298
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100299static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
300 size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301{
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -0800302 struct dm_path *path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100304 path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 if (!path)
306 return -ENXIO;
307
308 m->current_pgpath = path_to_pgpath(path);
309
310 if (m->current_pg != pg)
311 __switch_pg(m, m->current_pgpath);
312
313 return 0;
314}
315
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100316static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317{
318 struct priority_group *pg;
319 unsigned bypassed = 1;
320
Benjamin Marzinski1f271972014-08-13 13:53:42 -0500321 if (!m->nr_valid_paths) {
322 m->queue_io = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 goto failed;
Benjamin Marzinski1f271972014-08-13 13:53:42 -0500324 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325
326 /* Were we instructed to switch PG? */
327 if (m->next_pg) {
328 pg = m->next_pg;
329 m->next_pg = NULL;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100330 if (!__choose_path_in_pg(m, pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 return;
332 }
333
334 /* Don't change PG until it has no remaining paths */
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100335 if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 return;
337
338 /*
339 * Loop through priority groups until we find a valid path.
340 * First time we skip PGs marked 'bypassed'.
Mike Christief220fd42012-06-03 00:29:45 +0100341 * Second time we only try the ones we skipped, but set
342 * pg_init_delay_retry so we do not hammer controllers.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 */
344 do {
345 list_for_each_entry(pg, &m->priority_groups, list) {
346 if (pg->bypassed == bypassed)
347 continue;
Mike Christief220fd42012-06-03 00:29:45 +0100348 if (!__choose_path_in_pg(m, pg, nr_bytes)) {
349 if (!bypassed)
350 m->pg_init_delay_retry = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 return;
Mike Christief220fd42012-06-03 00:29:45 +0100352 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 }
354 } while (bypassed--);
355
356failed:
357 m->current_pgpath = NULL;
358 m->current_pg = NULL;
359}
360
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800361/*
362 * Check whether bios must be queued in the device-mapper core rather
363 * than here in the target.
364 *
365 * m->lock must be held on entry.
366 *
367 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
368 * same value then we are not between multipath_presuspend()
369 * and multipath_resume() calls and we have no need to check
370 * for the DMF_NOFLUSH_SUSPENDING flag.
371 */
372static int __must_push_back(struct multipath *m)
373{
Hannes Reineckee8099172014-02-28 15:33:44 +0100374 return (m->queue_if_no_path ||
375 (m->queue_if_no_path != m->saved_queue_if_no_path &&
376 dm_noflush_suspending(m->ti)));
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800377}
378
Hannes Reinecke36fcffc2014-02-28 15:33:47 +0100379/*
380 * Map cloned requests
381 */
Mike Snitzere5863d92014-12-17 21:08:12 -0500382static int __multipath_map(struct dm_target *ti, struct request *clone,
383 union map_info *map_context,
384 struct request *rq, struct request **__clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385{
Hannes Reinecke36fcffc2014-02-28 15:33:47 +0100386 struct multipath *m = (struct multipath *) ti->private;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100387 int r = DM_MAPIO_REQUEUE;
Mike Snitzere5863d92014-12-17 21:08:12 -0500388 size_t nr_bytes = clone ? blk_rq_bytes(clone) : blk_rq_bytes(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 struct pgpath *pgpath;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100390 struct block_device *bdev;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100391 struct dm_mpath_io *mpio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600393 spin_lock_irq(&m->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394
395 /* Do we need to select a new pgpath? */
396 if (!m->current_pgpath ||
397 (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100398 __choose_pgpath(m, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399
400 pgpath = m->current_pgpath;
401
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100402 if (!pgpath) {
403 if (!__must_push_back(m))
404 r = -EIO; /* Failed */
405 goto out_unlock;
Mike Snitzer6afbc012014-07-08 11:55:09 -0400406 } else if (m->queue_io || m->pg_init_required) {
Hannes Reineckee3bde042014-02-28 15:33:46 +0100407 __pg_init_all_paths(m);
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100408 goto out_unlock;
409 }
Mike Snitzer6afbc012014-07-08 11:55:09 -0400410
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100411 if (set_mapinfo(m, map_context) < 0)
412 /* ENOMEM, requeue */
413 goto out_unlock;
414
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100415 mpio = map_context->ptr;
416 mpio->pgpath = pgpath;
417 mpio->nr_bytes = nr_bytes;
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600418
419 bdev = pgpath->path.dev->bdev;
420
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600421 spin_unlock_irq(&m->lock);
422
Mike Snitzere5863d92014-12-17 21:08:12 -0500423 if (clone) {
424 /* Old request-based interface: allocated clone is passed in */
425 clone->q = bdev_get_queue(bdev);
426 clone->rq_disk = bdev->bd_disk;
427 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
428 } else {
429 /* blk-mq request-based interface */
430 *__clone = blk_get_request(bdev_get_queue(bdev),
Mike Snitzer02233342015-03-10 23:49:26 -0400431 rq_data_dir(rq), GFP_ATOMIC);
Mike Snitzere5863d92014-12-17 21:08:12 -0500432 if (IS_ERR(*__clone))
433 /* ENOMEM, requeue */
434 return r;
435 (*__clone)->bio = (*__clone)->biotail = NULL;
436 (*__clone)->rq_disk = bdev->bd_disk;
437 (*__clone)->cmd_flags |= REQ_FAILFAST_TRANSPORT;
438 }
439
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100440 if (pgpath->pg->ps.type->start_io)
441 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
442 &pgpath->path,
443 nr_bytes);
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600444 return DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445
Hannes Reineckee3bde042014-02-28 15:33:46 +0100446out_unlock:
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600447 spin_unlock_irq(&m->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448
449 return r;
450}
451
Mike Snitzere5863d92014-12-17 21:08:12 -0500452static int multipath_map(struct dm_target *ti, struct request *clone,
453 union map_info *map_context)
454{
455 return __multipath_map(ti, clone, map_context, NULL, NULL);
456}
457
458static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
459 union map_info *map_context,
460 struct request **clone)
461{
462 return __multipath_map(ti, NULL, map_context, rq, clone);
463}
464
465static void multipath_release_clone(struct request *clone)
466{
467 blk_put_request(clone);
468}
469
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470/*
471 * If we run out of usable paths, should we queue I/O or error it?
472 */
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700473static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
474 unsigned save_old_value)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475{
476 unsigned long flags;
477
478 spin_lock_irqsave(&m->lock, flags);
479
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700480 if (save_old_value)
481 m->saved_queue_if_no_path = m->queue_if_no_path;
482 else
483 m->saved_queue_if_no_path = queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 m->queue_if_no_path = queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 spin_unlock_irqrestore(&m->lock, flags);
486
Hannes Reinecke63d832c2014-05-26 14:45:39 +0200487 if (!queue_if_no_path)
488 dm_table_run_md_queue_async(m->ti->table);
489
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 return 0;
491}
492
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493/*
494 * An event is triggered whenever a path is taken out of use.
495 * Includes path failure and PG bypass.
496 */
David Howellsc4028952006-11-22 14:57:56 +0000497static void trigger_event(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498{
David Howellsc4028952006-11-22 14:57:56 +0000499 struct multipath *m =
500 container_of(work, struct multipath, trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501
502 dm_table_event(m->ti->table);
503}
504
505/*-----------------------------------------------------------------
506 * Constructor/argument parsing:
507 * <#multipath feature args> [<arg>]*
508 * <#hw_handler args> [hw_handler [<arg>]*]
509 * <#priority groups>
510 * <initial priority group>
511 * [<selector> <#selector args> [<arg>]*
512 * <#paths> <#per-path selector args>
513 * [<path> [<arg>]* ]+ ]+
514 *---------------------------------------------------------------*/
Mike Snitzer498f0102011-08-02 12:32:04 +0100515static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 struct dm_target *ti)
517{
518 int r;
519 struct path_selector_type *pst;
520 unsigned ps_argc;
521
Mike Snitzer498f0102011-08-02 12:32:04 +0100522 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700523 {0, 1024, "invalid number of path selector args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 };
525
Mike Snitzer498f0102011-08-02 12:32:04 +0100526 pst = dm_get_path_selector(dm_shift_arg(as));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 if (!pst) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700528 ti->error = "unknown path selector type";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 return -EINVAL;
530 }
531
Mike Snitzer498f0102011-08-02 12:32:04 +0100532 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100533 if (r) {
534 dm_put_path_selector(pst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 return -EINVAL;
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100536 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537
538 r = pst->create(&pg->ps, ps_argc, as->argv);
539 if (r) {
540 dm_put_path_selector(pst);
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700541 ti->error = "path selector constructor failed";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 return r;
543 }
544
545 pg->ps.type = pst;
Mike Snitzer498f0102011-08-02 12:32:04 +0100546 dm_consume_args(as, ps_argc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
548 return 0;
549}
550
Mike Snitzer498f0102011-08-02 12:32:04 +0100551static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 struct dm_target *ti)
553{
554 int r;
555 struct pgpath *p;
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700556 struct multipath *m = ti->private;
Mike Snitzera58a9352012-07-27 15:08:04 +0100557 struct request_queue *q = NULL;
558 const char *attached_handler_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559
560 /* we need at least a path arg */
561 if (as->argc < 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700562 ti->error = "no device given";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100563 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 }
565
566 p = alloc_pgpath();
567 if (!p)
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100568 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569
Mike Snitzer498f0102011-08-02 12:32:04 +0100570 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +0000571 &p->path.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700573 ti->error = "error getting device";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 goto bad;
575 }
576
Mike Snitzera58a9352012-07-27 15:08:04 +0100577 if (m->retain_attached_hw_handler || m->hw_handler_name)
578 q = bdev_get_queue(p->path.dev->bdev);
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100579
Mike Snitzera58a9352012-07-27 15:08:04 +0100580 if (m->retain_attached_hw_handler) {
581 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
582 if (attached_handler_name) {
583 /*
584 * Reset hw_handler_name to match the attached handler
585 * and clear any hw_handler_params associated with the
586 * ignored handler.
587 *
588 * NB. This modifies the table line to show the actual
589 * handler instead of the original table passed in.
590 */
591 kfree(m->hw_handler_name);
592 m->hw_handler_name = attached_handler_name;
593
594 kfree(m->hw_handler_params);
595 m->hw_handler_params = NULL;
596 }
597 }
598
599 if (m->hw_handler_name) {
600 /*
601 * Increments scsi_dh reference, even when using an
602 * already-attached handler.
603 */
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100604 r = scsi_dh_attach(q, m->hw_handler_name);
605 if (r == -EBUSY) {
606 /*
Mike Snitzera58a9352012-07-27 15:08:04 +0100607 * Already attached to different hw_handler:
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100608 * try to reattach with correct one.
609 */
610 scsi_dh_detach(q);
611 r = scsi_dh_attach(q, m->hw_handler_name);
612 }
613
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700614 if (r < 0) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100615 ti->error = "error attaching hardware handler";
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700616 dm_put_device(ti, p->path.dev);
617 goto bad;
618 }
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700619
620 if (m->hw_handler_params) {
621 r = scsi_dh_set_params(q, m->hw_handler_params);
622 if (r < 0) {
623 ti->error = "unable to set hardware "
624 "handler parameters";
625 scsi_dh_detach(q);
626 dm_put_device(ti, p->path.dev);
627 goto bad;
628 }
629 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700630 }
631
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
633 if (r) {
634 dm_put_device(ti, p->path.dev);
635 goto bad;
636 }
637
638 return p;
639
640 bad:
641 free_pgpath(p);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100642 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643}
644
Mike Snitzer498f0102011-08-02 12:32:04 +0100645static struct priority_group *parse_priority_group(struct dm_arg_set *as,
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700646 struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647{
Mike Snitzer498f0102011-08-02 12:32:04 +0100648 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700649 {1, 1024, "invalid number of paths"},
650 {0, 1024, "invalid number of selector args"}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 };
652
653 int r;
Mike Snitzer498f0102011-08-02 12:32:04 +0100654 unsigned i, nr_selector_args, nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 struct priority_group *pg;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700656 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657
658 if (as->argc < 2) {
659 as->argc = 0;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100660 ti->error = "not enough priority group arguments";
661 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 }
663
664 pg = alloc_priority_group();
665 if (!pg) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700666 ti->error = "couldn't allocate priority group";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100667 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 }
669 pg->m = m;
670
671 r = parse_path_selector(as, pg, ti);
672 if (r)
673 goto bad;
674
675 /*
676 * read the paths
677 */
Mike Snitzer498f0102011-08-02 12:32:04 +0100678 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 if (r)
680 goto bad;
681
Mike Snitzer498f0102011-08-02 12:32:04 +0100682 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 if (r)
684 goto bad;
685
Mike Snitzer498f0102011-08-02 12:32:04 +0100686 nr_args = 1 + nr_selector_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 for (i = 0; i < pg->nr_pgpaths; i++) {
688 struct pgpath *pgpath;
Mike Snitzer498f0102011-08-02 12:32:04 +0100689 struct dm_arg_set path_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690
Mike Snitzer498f0102011-08-02 12:32:04 +0100691 if (as->argc < nr_args) {
Mikulas Patocka148acff2008-07-21 12:00:30 +0100692 ti->error = "not enough path parameters";
Alasdair G Kergon6bbf79a2010-08-12 04:13:49 +0100693 r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 goto bad;
Mikulas Patocka148acff2008-07-21 12:00:30 +0100695 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696
Mike Snitzer498f0102011-08-02 12:32:04 +0100697 path_args.argc = nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 path_args.argv = as->argv;
699
700 pgpath = parse_path(&path_args, &pg->ps, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100701 if (IS_ERR(pgpath)) {
702 r = PTR_ERR(pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 goto bad;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100704 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705
706 pgpath->pg = pg;
707 list_add_tail(&pgpath->list, &pg->pgpaths);
Mike Snitzer498f0102011-08-02 12:32:04 +0100708 dm_consume_args(as, nr_args);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 }
710
711 return pg;
712
713 bad:
714 free_priority_group(pg, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100715 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716}
717
Mike Snitzer498f0102011-08-02 12:32:04 +0100718static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 unsigned hw_argc;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700721 int ret;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700722 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723
Mike Snitzer498f0102011-08-02 12:32:04 +0100724 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700725 {0, 1024, "invalid number of hardware handler args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 };
727
Mike Snitzer498f0102011-08-02 12:32:04 +0100728 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 return -EINVAL;
730
731 if (!hw_argc)
732 return 0;
733
Mike Snitzer498f0102011-08-02 12:32:04 +0100734 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
Mike Snitzer510193a2012-05-12 01:43:21 +0100735 if (!try_then_request_module(scsi_dh_handler_exist(m->hw_handler_name),
736 "scsi_dh_%s", m->hw_handler_name)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700737 ti->error = "unknown hardware handler type";
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700738 ret = -EINVAL;
739 goto fail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 }
Chandra Seetharaman14e98c52008-11-13 23:39:06 +0000741
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700742 if (hw_argc > 1) {
743 char *p;
744 int i, j, len = 4;
745
746 for (i = 0; i <= hw_argc - 2; i++)
747 len += strlen(as->argv[i]) + 1;
748 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
749 if (!p) {
750 ti->error = "memory allocation failed";
751 ret = -ENOMEM;
752 goto fail;
753 }
754 j = sprintf(p, "%d", hw_argc - 1);
755 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
756 j = sprintf(p, "%s", as->argv[i]);
757 }
Mike Snitzer498f0102011-08-02 12:32:04 +0100758 dm_consume_args(as, hw_argc - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759
760 return 0;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700761fail:
762 kfree(m->hw_handler_name);
763 m->hw_handler_name = NULL;
764 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765}
766
Mike Snitzer498f0102011-08-02 12:32:04 +0100767static int parse_features(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768{
769 int r;
770 unsigned argc;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700771 struct dm_target *ti = m->ti;
Mike Snitzer498f0102011-08-02 12:32:04 +0100772 const char *arg_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773
Mike Snitzer498f0102011-08-02 12:32:04 +0100774 static struct dm_arg _args[] = {
Mike Snitzera58a9352012-07-27 15:08:04 +0100775 {0, 6, "invalid number of feature args"},
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100776 {1, 50, "pg_init_retries must be between 1 and 50"},
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000777 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 };
779
Mike Snitzer498f0102011-08-02 12:32:04 +0100780 r = dm_read_arg_group(_args, as, &argc, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 if (r)
782 return -EINVAL;
783
784 if (!argc)
785 return 0;
786
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100787 do {
Mike Snitzer498f0102011-08-02 12:32:04 +0100788 arg_name = dm_shift_arg(as);
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100789 argc--;
790
Mike Snitzer498f0102011-08-02 12:32:04 +0100791 if (!strcasecmp(arg_name, "queue_if_no_path")) {
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100792 r = queue_if_no_path(m, 1, 0);
793 continue;
794 }
795
Mike Snitzera58a9352012-07-27 15:08:04 +0100796 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
797 m->retain_attached_hw_handler = 1;
798 continue;
799 }
800
Mike Snitzer498f0102011-08-02 12:32:04 +0100801 if (!strcasecmp(arg_name, "pg_init_retries") &&
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100802 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +0100803 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100804 argc--;
805 continue;
806 }
807
Mike Snitzer498f0102011-08-02 12:32:04 +0100808 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000809 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +0100810 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000811 argc--;
812 continue;
813 }
814
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 ti->error = "Unrecognised multipath feature request";
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100816 r = -EINVAL;
817 } while (argc && !r);
818
819 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820}
821
822static int multipath_ctr(struct dm_target *ti, unsigned int argc,
823 char **argv)
824{
Mike Snitzer498f0102011-08-02 12:32:04 +0100825 /* target arguments */
826 static struct dm_arg _args[] = {
Mike Snitzera490a072011-03-24 13:54:33 +0000827 {0, 1024, "invalid number of priority groups"},
828 {0, 1024, "invalid initial priority group number"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 };
830
831 int r;
832 struct multipath *m;
Mike Snitzer498f0102011-08-02 12:32:04 +0100833 struct dm_arg_set as;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 unsigned pg_count = 0;
835 unsigned next_pg_num;
836
837 as.argc = argc;
838 as.argv = argv;
839
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700840 m = alloc_multipath(ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 if (!m) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700842 ti->error = "can't allocate multipath";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843 return -EINVAL;
844 }
845
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700846 r = parse_features(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 if (r)
848 goto bad;
849
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700850 r = parse_hw_handler(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 if (r)
852 goto bad;
853
Mike Snitzer498f0102011-08-02 12:32:04 +0100854 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 if (r)
856 goto bad;
857
Mike Snitzer498f0102011-08-02 12:32:04 +0100858 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 if (r)
860 goto bad;
861
Mike Snitzera490a072011-03-24 13:54:33 +0000862 if ((!m->nr_priority_groups && next_pg_num) ||
863 (m->nr_priority_groups && !next_pg_num)) {
864 ti->error = "invalid initial priority group";
865 r = -EINVAL;
866 goto bad;
867 }
868
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 /* parse the priority groups */
870 while (as.argc) {
871 struct priority_group *pg;
872
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700873 pg = parse_priority_group(&as, m);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100874 if (IS_ERR(pg)) {
875 r = PTR_ERR(pg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 goto bad;
877 }
878
879 m->nr_valid_paths += pg->nr_pgpaths;
880 list_add_tail(&pg->list, &m->priority_groups);
881 pg_count++;
882 pg->pg_num = pg_count;
883 if (!--next_pg_num)
884 m->next_pg = pg;
885 }
886
887 if (pg_count != m->nr_priority_groups) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700888 ti->error = "priority group count mismatch";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 r = -EINVAL;
890 goto bad;
891 }
892
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +0000893 ti->num_flush_bios = 1;
894 ti->num_discard_bios = 1;
Mike Snitzer042bcef2013-05-10 14:37:16 +0100895 ti->num_write_same_bios = 1;
Mikulas Patocka86279212009-06-22 10:12:24 +0100896
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 return 0;
898
899 bad:
900 free_multipath(m);
901 return r;
902}
903
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000904static void multipath_wait_for_pg_init_completion(struct multipath *m)
905{
906 DECLARE_WAITQUEUE(wait, current);
907 unsigned long flags;
908
909 add_wait_queue(&m->pg_init_wait, &wait);
910
911 while (1) {
912 set_current_state(TASK_UNINTERRUPTIBLE);
913
914 spin_lock_irqsave(&m->lock, flags);
915 if (!m->pg_init_in_progress) {
916 spin_unlock_irqrestore(&m->lock, flags);
917 break;
918 }
919 spin_unlock_irqrestore(&m->lock, flags);
920
921 io_schedule();
922 }
923 set_current_state(TASK_RUNNING);
924
925 remove_wait_queue(&m->pg_init_wait, &wait);
926}
927
928static void flush_multipath_work(struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929{
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000930 unsigned long flags;
931
932 spin_lock_irqsave(&m->lock, flags);
933 m->pg_init_disabled = 1;
934 spin_unlock_irqrestore(&m->lock, flags);
935
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700936 flush_workqueue(kmpath_handlerd);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000937 multipath_wait_for_pg_init_completion(m);
Alasdair G Kergona044d012005-07-12 15:53:02 -0700938 flush_workqueue(kmultipathd);
Tejun Heo43829732012-08-20 14:51:24 -0700939 flush_work(&m->trigger_event);
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000940
941 spin_lock_irqsave(&m->lock, flags);
942 m->pg_init_disabled = 0;
943 spin_unlock_irqrestore(&m->lock, flags);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +0000944}
945
946static void multipath_dtr(struct dm_target *ti)
947{
948 struct multipath *m = ti->private;
949
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000950 flush_multipath_work(m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 free_multipath(m);
952}
953
954/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955 * Take a path out of use.
956 */
957static int fail_path(struct pgpath *pgpath)
958{
959 unsigned long flags;
960 struct multipath *m = pgpath->pg->m;
961
962 spin_lock_irqsave(&m->lock, flags);
963
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100964 if (!pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 goto out;
966
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700967 DMWARN("Failing path %s.", pgpath->path.dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968
969 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100970 pgpath->is_active = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971 pgpath->fail_count++;
972
973 m->nr_valid_paths--;
974
975 if (pgpath == m->current_pgpath)
976 m->current_pgpath = NULL;
977
Mike Andersonb15546f2007-10-19 22:48:02 +0100978 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
979 pgpath->path.dev->name, m->nr_valid_paths);
980
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +0000981 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
983out:
984 spin_unlock_irqrestore(&m->lock, flags);
985
986 return 0;
987}
988
989/*
990 * Reinstate a previously-failed path
991 */
992static int reinstate_path(struct pgpath *pgpath)
993{
Hannes Reinecke63d832c2014-05-26 14:45:39 +0200994 int r = 0, run_queue = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 unsigned long flags;
996 struct multipath *m = pgpath->pg->m;
997
998 spin_lock_irqsave(&m->lock, flags);
999
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001000 if (pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 goto out;
1002
Alasdair G Kergondef052d2008-07-21 12:00:31 +01001003 if (!pgpath->pg->ps.type->reinstate_path) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 DMWARN("Reinstate path not supported by path selector %s",
1005 pgpath->pg->ps.type->name);
1006 r = -EINVAL;
1007 goto out;
1008 }
1009
1010 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1011 if (r)
1012 goto out;
1013
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001014 pgpath->is_active = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
Hannes Reineckee8099172014-02-28 15:33:44 +01001016 if (!m->nr_valid_paths++) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001017 m->current_pgpath = NULL;
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001018 run_queue = 1;
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001019 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001020 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001021 m->pg_init_in_progress++;
1022 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023
Mike Andersonb15546f2007-10-19 22:48:02 +01001024 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1025 pgpath->path.dev->name, m->nr_valid_paths);
1026
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001027 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028
1029out:
1030 spin_unlock_irqrestore(&m->lock, flags);
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001031 if (run_queue)
1032 dm_table_run_md_queue_async(m->ti->table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033
1034 return r;
1035}
1036
1037/*
1038 * Fail or reinstate all paths that match the provided struct dm_dev.
1039 */
1040static int action_dev(struct multipath *m, struct dm_dev *dev,
1041 action_fn action)
1042{
Mike Snitzer19040c02011-03-24 13:54:31 +00001043 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044 struct pgpath *pgpath;
1045 struct priority_group *pg;
1046
1047 list_for_each_entry(pg, &m->priority_groups, list) {
1048 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1049 if (pgpath->path.dev == dev)
1050 r = action(pgpath);
1051 }
1052 }
1053
1054 return r;
1055}
1056
1057/*
1058 * Temporarily try to avoid having to use the specified PG
1059 */
1060static void bypass_pg(struct multipath *m, struct priority_group *pg,
1061 int bypassed)
1062{
1063 unsigned long flags;
1064
1065 spin_lock_irqsave(&m->lock, flags);
1066
1067 pg->bypassed = bypassed;
1068 m->current_pgpath = NULL;
1069 m->current_pg = NULL;
1070
1071 spin_unlock_irqrestore(&m->lock, flags);
1072
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001073 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074}
1075
1076/*
1077 * Switch to using the specified PG from the next I/O that gets mapped
1078 */
1079static int switch_pg_num(struct multipath *m, const char *pgstr)
1080{
1081 struct priority_group *pg;
1082 unsigned pgnum;
1083 unsigned long flags;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001084 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001086 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087 (pgnum > m->nr_priority_groups)) {
1088 DMWARN("invalid PG number supplied to switch_pg_num");
1089 return -EINVAL;
1090 }
1091
1092 spin_lock_irqsave(&m->lock, flags);
1093 list_for_each_entry(pg, &m->priority_groups, list) {
1094 pg->bypassed = 0;
1095 if (--pgnum)
1096 continue;
1097
1098 m->current_pgpath = NULL;
1099 m->current_pg = NULL;
1100 m->next_pg = pg;
1101 }
1102 spin_unlock_irqrestore(&m->lock, flags);
1103
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001104 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 return 0;
1106}
1107
1108/*
1109 * Set/clear bypassed status of a PG.
1110 * PGs are numbered upwards from 1 in the order they were declared.
1111 */
1112static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
1113{
1114 struct priority_group *pg;
1115 unsigned pgnum;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001116 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001118 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119 (pgnum > m->nr_priority_groups)) {
1120 DMWARN("invalid PG number supplied to bypass_pg");
1121 return -EINVAL;
1122 }
1123
1124 list_for_each_entry(pg, &m->priority_groups, list) {
1125 if (!--pgnum)
1126 break;
1127 }
1128
1129 bypass_pg(m, pg, bypassed);
1130 return 0;
1131}
1132
1133/*
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001134 * Should we retry pg_init immediately?
1135 */
1136static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1137{
1138 unsigned long flags;
1139 int limit_reached = 0;
1140
1141 spin_lock_irqsave(&m->lock, flags);
1142
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +00001143 if (m->pg_init_count <= m->pg_init_retries && !m->pg_init_disabled)
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001144 m->pg_init_required = 1;
1145 else
1146 limit_reached = 1;
1147
1148 spin_unlock_irqrestore(&m->lock, flags);
1149
1150 return limit_reached;
1151}
1152
Chandra Seetharaman3ae31f62009-10-21 09:22:46 -07001153static void pg_init_done(void *data, int errors)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001154{
Moger, Babu83c0d5d2010-03-06 02:29:45 +00001155 struct pgpath *pgpath = data;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001156 struct priority_group *pg = pgpath->pg;
1157 struct multipath *m = pg->m;
1158 unsigned long flags;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001159 unsigned delay_retry = 0;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001160
1161 /* device or driver problems */
1162 switch (errors) {
1163 case SCSI_DH_OK:
1164 break;
1165 case SCSI_DH_NOSYS:
1166 if (!m->hw_handler_name) {
1167 errors = 0;
1168 break;
1169 }
Moger, Babuf7b934c2010-03-06 02:29:49 +00001170 DMERR("Could not failover the device: Handler scsi_dh_%s "
1171 "Error %d.", m->hw_handler_name, errors);
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001172 /*
1173 * Fail path for now, so we do not ping pong
1174 */
1175 fail_path(pgpath);
1176 break;
1177 case SCSI_DH_DEV_TEMP_BUSY:
1178 /*
1179 * Probably doing something like FW upgrade on the
1180 * controller so try the other pg.
1181 */
1182 bypass_pg(m, pg, 1);
1183 break;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001184 case SCSI_DH_RETRY:
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001185 /* Wait before retrying. */
1186 delay_retry = 1;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001187 case SCSI_DH_IMM_RETRY:
1188 case SCSI_DH_RES_TEMP_UNAVAIL:
1189 if (pg_init_limit_reached(m, pgpath))
1190 fail_path(pgpath);
1191 errors = 0;
1192 break;
1193 default:
1194 /*
1195 * We probably do not want to fail the path for a device
1196 * error, but this is what the old dm did. In future
1197 * patches we can do more advanced handling.
1198 */
1199 fail_path(pgpath);
1200 }
1201
1202 spin_lock_irqsave(&m->lock, flags);
1203 if (errors) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001204 if (pgpath == m->current_pgpath) {
1205 DMERR("Could not failover device. Error %d.", errors);
1206 m->current_pgpath = NULL;
1207 m->current_pg = NULL;
1208 }
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001209 } else if (!m->pg_init_required)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001210 pg->bypassed = 0;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001211
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001212 if (--m->pg_init_in_progress)
1213 /* Activations of other paths are still on going */
1214 goto out;
1215
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001216 if (m->pg_init_required) {
1217 m->pg_init_delay_retry = delay_retry;
1218 if (__pg_init_all_paths(m))
1219 goto out;
1220 }
1221 m->queue_io = 0;
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001222
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001223 /*
1224 * Wake up any thread waiting to suspend.
1225 */
1226 wake_up(&m->pg_init_wait);
1227
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001228out:
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001229 spin_unlock_irqrestore(&m->lock, flags);
1230}
1231
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001232static void activate_path(struct work_struct *work)
1233{
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001234 struct pgpath *pgpath =
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001235 container_of(work, struct pgpath, activate_path.work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001236
Hannes Reinecke3a017502014-02-28 15:33:49 +01001237 if (pgpath->is_active)
1238 scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
1239 pg_init_done, pgpath);
1240 else
1241 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001242}
1243
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001244static int noretry_error(int error)
1245{
1246 switch (error) {
1247 case -EOPNOTSUPP:
1248 case -EREMOTEIO:
1249 case -EILSEQ:
1250 case -ENODATA:
Jun'ichi Nomuracc9d3c32013-09-13 14:54:30 +09001251 case -ENOSPC:
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001252 return 1;
1253 }
1254
1255 /* Anything else could be a path failure, so should be retried */
1256 return 0;
1257}
1258
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259/*
1260 * end_io handling
1261 */
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001262static int do_end_io(struct multipath *m, struct request *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001263 int error, struct dm_mpath_io *mpio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264{
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001265 /*
1266 * We don't queue any clone request inside the multipath target
1267 * during end I/O handling, since those clone requests don't have
1268 * bio clones. If we queue them inside the multipath target,
1269 * we need to make bio clones, that requires memory allocation.
1270 * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
1271 * don't have bio clones.)
1272 * Instead of queueing the clone request here, we queue the original
1273 * request into dm core, which will remake a clone request and
1274 * clone bios for it and resubmit it later.
1275 */
1276 int r = DM_ENDIO_REQUEUE;
Stefan Bader640eb3b2005-11-21 21:32:35 -08001277 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001278
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001279 if (!error && !clone->errors)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280 return 0; /* I/O complete */
1281
Mike Snitzer7eee4ae2014-06-02 15:50:06 -04001282 if (noretry_error(error))
Mike Snitzer959eb4e2010-08-12 04:14:32 +01001283 return error;
1284
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001285 if (mpio->pgpath)
1286 fail_path(mpio->pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287
Stefan Bader640eb3b2005-11-21 21:32:35 -08001288 spin_lock_irqsave(&m->lock, flags);
Hannes Reinecke751b2a72011-01-18 10:13:12 +01001289 if (!m->nr_valid_paths) {
1290 if (!m->queue_if_no_path) {
1291 if (!__must_push_back(m))
1292 r = -EIO;
1293 } else {
1294 if (error == -EBADE)
1295 r = error;
1296 }
1297 }
Stefan Bader640eb3b2005-11-21 21:32:35 -08001298 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001300 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301}
1302
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001303static int multipath_end_io(struct dm_target *ti, struct request *clone,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304 int error, union map_info *map_context)
1305{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001306 struct multipath *m = ti->private;
1307 struct dm_mpath_io *mpio = map_context->ptr;
Wei Yongjuna71a2612012-10-12 16:59:42 +01001308 struct pgpath *pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309 struct path_selector *ps;
1310 int r;
1311
Jun'ichi Nomura466891f2012-03-28 18:41:25 +01001312 BUG_ON(!mpio);
1313
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001314 r = do_end_io(m, clone, error, mpio);
Wei Yongjuna71a2612012-10-12 16:59:42 +01001315 pgpath = mpio->pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316 if (pgpath) {
1317 ps = &pgpath->pg->ps;
1318 if (ps->type->end_io)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001319 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320 }
Jun'ichi Nomura466891f2012-03-28 18:41:25 +01001321 clear_mapinfo(m, map_context);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322
1323 return r;
1324}
1325
1326/*
1327 * Suspend can't complete until all the I/O is processed so if
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001328 * the last path fails we must error any remaining I/O.
1329 * Note that if the freeze_bdev fails while suspending, the
1330 * queue_if_no_path state is lost - userspace should reset it.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331 */
1332static void multipath_presuspend(struct dm_target *ti)
1333{
1334 struct multipath *m = (struct multipath *) ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335
Alasdair G Kergon485ef692005-09-27 21:45:45 -07001336 queue_if_no_path(m, 0, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337}
1338
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001339static void multipath_postsuspend(struct dm_target *ti)
1340{
Mike Anderson6380f262009-12-10 23:52:21 +00001341 struct multipath *m = ti->private;
1342
1343 mutex_lock(&m->work_mutex);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001344 flush_multipath_work(m);
Mike Anderson6380f262009-12-10 23:52:21 +00001345 mutex_unlock(&m->work_mutex);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001346}
1347
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001348/*
1349 * Restore the queue_if_no_path setting.
1350 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351static void multipath_resume(struct dm_target *ti)
1352{
1353 struct multipath *m = (struct multipath *) ti->private;
1354 unsigned long flags;
1355
1356 spin_lock_irqsave(&m->lock, flags);
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001357 m->queue_if_no_path = m->saved_queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358 spin_unlock_irqrestore(&m->lock, flags);
1359}
1360
1361/*
1362 * Info output has the following format:
1363 * num_multipath_feature_args [multipath_feature_args]*
1364 * num_handler_status_args [handler_status_args]*
1365 * num_groups init_group_number
1366 * [A|D|E num_ps_status_args [ps_status_args]*
1367 * num_paths num_selector_args
1368 * [path_dev A|F fail_count [selector_args]* ]+ ]+
1369 *
1370 * Table output has the following format (identical to the constructor string):
1371 * num_feature_args [features_args]*
1372 * num_handler_args hw_handler [hw_handler_args]*
1373 * num_groups init_group_number
1374 * [priority selector-name num_ps_args [ps_args]*
1375 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1376 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00001377static void multipath_status(struct dm_target *ti, status_type_t type,
1378 unsigned status_flags, char *result, unsigned maxlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379{
1380 int sz = 0;
1381 unsigned long flags;
1382 struct multipath *m = (struct multipath *) ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 struct priority_group *pg;
1384 struct pgpath *p;
1385 unsigned pg_num;
1386 char state;
1387
1388 spin_lock_irqsave(&m->lock, flags);
1389
1390 /* Features */
1391 if (type == STATUSTYPE_INFO)
Hannes Reineckee8099172014-02-28 15:33:44 +01001392 DMEMIT("2 %u %u ", m->queue_io, m->pg_init_count);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001393 else {
1394 DMEMIT("%u ", m->queue_if_no_path +
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001395 (m->pg_init_retries > 0) * 2 +
Mike Snitzera58a9352012-07-27 15:08:04 +01001396 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1397 m->retain_attached_hw_handler);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001398 if (m->queue_if_no_path)
1399 DMEMIT("queue_if_no_path ");
1400 if (m->pg_init_retries)
1401 DMEMIT("pg_init_retries %u ", m->pg_init_retries);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001402 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1403 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
Mike Snitzera58a9352012-07-27 15:08:04 +01001404 if (m->retain_attached_hw_handler)
1405 DMEMIT("retain_attached_hw_handler ");
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001406 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001408 if (!m->hw_handler_name || type == STATUSTYPE_INFO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409 DMEMIT("0 ");
1410 else
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001411 DMEMIT("1 %s ", m->hw_handler_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001412
1413 DMEMIT("%u ", m->nr_priority_groups);
1414
1415 if (m->next_pg)
1416 pg_num = m->next_pg->pg_num;
1417 else if (m->current_pg)
1418 pg_num = m->current_pg->pg_num;
1419 else
Mike Snitzera490a072011-03-24 13:54:33 +00001420 pg_num = (m->nr_priority_groups ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421
1422 DMEMIT("%u ", pg_num);
1423
1424 switch (type) {
1425 case STATUSTYPE_INFO:
1426 list_for_each_entry(pg, &m->priority_groups, list) {
1427 if (pg->bypassed)
1428 state = 'D'; /* Disabled */
1429 else if (pg == m->current_pg)
1430 state = 'A'; /* Currently Active */
1431 else
1432 state = 'E'; /* Enabled */
1433
1434 DMEMIT("%c ", state);
1435
1436 if (pg->ps.type->status)
1437 sz += pg->ps.type->status(&pg->ps, NULL, type,
1438 result + sz,
1439 maxlen - sz);
1440 else
1441 DMEMIT("0 ");
1442
1443 DMEMIT("%u %u ", pg->nr_pgpaths,
1444 pg->ps.type->info_args);
1445
1446 list_for_each_entry(p, &pg->pgpaths, list) {
1447 DMEMIT("%s %s %u ", p->path.dev->name,
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001448 p->is_active ? "A" : "F",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001449 p->fail_count);
1450 if (pg->ps.type->status)
1451 sz += pg->ps.type->status(&pg->ps,
1452 &p->path, type, result + sz,
1453 maxlen - sz);
1454 }
1455 }
1456 break;
1457
1458 case STATUSTYPE_TABLE:
1459 list_for_each_entry(pg, &m->priority_groups, list) {
1460 DMEMIT("%s ", pg->ps.type->name);
1461
1462 if (pg->ps.type->status)
1463 sz += pg->ps.type->status(&pg->ps, NULL, type,
1464 result + sz,
1465 maxlen - sz);
1466 else
1467 DMEMIT("0 ");
1468
1469 DMEMIT("%u %u ", pg->nr_pgpaths,
1470 pg->ps.type->table_args);
1471
1472 list_for_each_entry(p, &pg->pgpaths, list) {
1473 DMEMIT("%s ", p->path.dev->name);
1474 if (pg->ps.type->status)
1475 sz += pg->ps.type->status(&pg->ps,
1476 &p->path, type, result + sz,
1477 maxlen - sz);
1478 }
1479 }
1480 break;
1481 }
1482
1483 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484}
1485
1486static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1487{
Mike Anderson6380f262009-12-10 23:52:21 +00001488 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 struct dm_dev *dev;
1490 struct multipath *m = (struct multipath *) ti->private;
1491 action_fn action;
1492
Mike Anderson6380f262009-12-10 23:52:21 +00001493 mutex_lock(&m->work_mutex);
1494
Kiyoshi Uedac2f3d242009-12-10 23:52:27 +00001495 if (dm_suspended(ti)) {
1496 r = -EBUSY;
1497 goto out;
1498 }
1499
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 if (argc == 1) {
Mike Snitzer498f0102011-08-02 12:32:04 +01001501 if (!strcasecmp(argv[0], "queue_if_no_path")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001502 r = queue_if_no_path(m, 1, 0);
1503 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001504 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001505 r = queue_if_no_path(m, 0, 0);
1506 goto out;
1507 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508 }
1509
Mike Anderson6380f262009-12-10 23:52:21 +00001510 if (argc != 2) {
Jose Castilloa356e422014-01-29 17:52:45 +01001511 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
Mike Anderson6380f262009-12-10 23:52:21 +00001512 goto out;
1513 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514
Mike Snitzer498f0102011-08-02 12:32:04 +01001515 if (!strcasecmp(argv[0], "disable_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001516 r = bypass_pg_num(m, argv[1], 1);
1517 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001518 } else if (!strcasecmp(argv[0], "enable_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001519 r = bypass_pg_num(m, argv[1], 0);
1520 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001521 } else if (!strcasecmp(argv[0], "switch_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001522 r = switch_pg_num(m, argv[1]);
1523 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001524 } else if (!strcasecmp(argv[0], "reinstate_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525 action = reinstate_path;
Mike Snitzer498f0102011-08-02 12:32:04 +01001526 else if (!strcasecmp(argv[0], "fail_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527 action = fail_path;
Mike Anderson6380f262009-12-10 23:52:21 +00001528 else {
Jose Castilloa356e422014-01-29 17:52:45 +01001529 DMWARN("Unrecognised multipath message received: %s", argv[0]);
Mike Anderson6380f262009-12-10 23:52:21 +00001530 goto out;
1531 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +00001533 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001535 DMWARN("message: error getting device %s",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001536 argv[1]);
Mike Anderson6380f262009-12-10 23:52:21 +00001537 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 }
1539
1540 r = action_dev(m, dev, action);
1541
1542 dm_put_device(ti, dev);
1543
Mike Anderson6380f262009-12-10 23:52:21 +00001544out:
1545 mutex_unlock(&m->work_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547}
1548
Al Viro647b3d02007-08-28 22:15:59 -04001549static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
Milan Broz9af4aa32006-10-03 01:15:20 -07001550 unsigned long arg)
1551{
Mikulas Patocka35991652012-06-03 00:29:58 +01001552 struct multipath *m = ti->private;
Mike Snitzer7ba10aa2012-09-26 23:45:41 +01001553 struct pgpath *pgpath;
Mikulas Patocka35991652012-06-03 00:29:58 +01001554 struct block_device *bdev;
1555 fmode_t mode;
Milan Broz9af4aa32006-10-03 01:15:20 -07001556 unsigned long flags;
Mikulas Patocka35991652012-06-03 00:29:58 +01001557 int r;
1558
Mikulas Patocka35991652012-06-03 00:29:58 +01001559 bdev = NULL;
1560 mode = 0;
1561 r = 0;
Milan Broz9af4aa32006-10-03 01:15:20 -07001562
1563 spin_lock_irqsave(&m->lock, flags);
1564
1565 if (!m->current_pgpath)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001566 __choose_pgpath(m, 0);
Milan Broz9af4aa32006-10-03 01:15:20 -07001567
Mike Snitzer7ba10aa2012-09-26 23:45:41 +01001568 pgpath = m->current_pgpath;
1569
1570 if (pgpath) {
1571 bdev = pgpath->path.dev->bdev;
1572 mode = pgpath->path.dev->mode;
Milan Broze90dae12006-10-03 01:15:22 -07001573 }
Milan Broz9af4aa32006-10-03 01:15:20 -07001574
Mike Snitzer7ba10aa2012-09-26 23:45:41 +01001575 if ((pgpath && m->queue_io) || (!pgpath && m->queue_if_no_path))
Hannes Reinecke6c182cd2013-07-10 23:41:15 +01001576 r = -ENOTCONN;
Milan Broz9af4aa32006-10-03 01:15:20 -07001577 else if (!bdev)
1578 r = -EIO;
1579
1580 spin_unlock_irqrestore(&m->lock, flags);
1581
Paolo Bonziniec8013b2012-01-12 16:01:29 +01001582 /*
1583 * Only pass ioctls through if the device sizes match exactly.
1584 */
Hannes Reineckea1989b32014-02-26 10:07:04 +01001585 if (!bdev || ti->len != i_size_read(bdev->bd_inode) >> SECTOR_SHIFT) {
1586 int err = scsi_verify_blk_ioctl(NULL, cmd);
1587 if (err)
1588 r = err;
1589 }
Paolo Bonziniec8013b2012-01-12 16:01:29 +01001590
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001591 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
1592 spin_lock_irqsave(&m->lock, flags);
1593 if (!m->current_pg) {
1594 /* Path status changed, redo selection */
1595 __choose_pgpath(m, 0);
1596 }
1597 if (m->pg_init_required)
1598 __pg_init_all_paths(m);
Mike Snitzer4cdd2ad2014-05-13 13:49:39 -04001599 spin_unlock_irqrestore(&m->lock, flags);
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001600 dm_table_run_md_queue_async(m->ti->table);
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001601 }
Mikulas Patocka35991652012-06-03 00:29:58 +01001602
Al Viro633a08b2007-08-29 20:34:12 -04001603 return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
Milan Broz9af4aa32006-10-03 01:15:20 -07001604}
1605
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001606static int multipath_iterate_devices(struct dm_target *ti,
1607 iterate_devices_callout_fn fn, void *data)
1608{
1609 struct multipath *m = ti->private;
1610 struct priority_group *pg;
1611 struct pgpath *p;
1612 int ret = 0;
1613
1614 list_for_each_entry(pg, &m->priority_groups, list) {
1615 list_for_each_entry(p, &pg->pgpaths, list) {
Mike Snitzer5dea2712009-07-23 20:30:42 +01001616 ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001617 if (ret)
1618 goto out;
1619 }
1620 }
1621
1622out:
1623 return ret;
1624}
1625
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001626static int __pgpath_busy(struct pgpath *pgpath)
1627{
1628 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1629
Mike Snitzer52b09912015-02-23 16:36:41 -05001630 return blk_lld_busy(q);
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001631}
1632
1633/*
1634 * We return "busy", only when we can map I/Os but underlying devices
1635 * are busy (so even if we map I/Os now, the I/Os will wait on
1636 * the underlying queue).
1637 * In other words, if we want to kill I/Os or queue them inside us
1638 * due to map unavailability, we don't return "busy". Otherwise,
1639 * dm core won't give us the I/Os and we can't do what we want.
1640 */
1641static int multipath_busy(struct dm_target *ti)
1642{
1643 int busy = 0, has_active = 0;
1644 struct multipath *m = ti->private;
1645 struct priority_group *pg;
1646 struct pgpath *pgpath;
1647 unsigned long flags;
1648
1649 spin_lock_irqsave(&m->lock, flags);
1650
Jun'ichi Nomura7a7a3b42014-07-08 00:55:14 +00001651 /* pg_init in progress or no paths available */
1652 if (m->pg_init_in_progress ||
1653 (!m->nr_valid_paths && m->queue_if_no_path)) {
Hannes Reineckeb63349a2013-10-01 11:49:56 +02001654 busy = 1;
1655 goto out;
1656 }
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001657 /* Guess which priority_group will be used at next mapping time */
1658 if (unlikely(!m->current_pgpath && m->next_pg))
1659 pg = m->next_pg;
1660 else if (likely(m->current_pg))
1661 pg = m->current_pg;
1662 else
1663 /*
1664 * We don't know which pg will be used at next mapping time.
1665 * We don't call __choose_pgpath() here to avoid to trigger
1666 * pg_init just by busy checking.
1667 * So we don't know whether underlying devices we will be using
1668 * at next mapping time are busy or not. Just try mapping.
1669 */
1670 goto out;
1671
1672 /*
1673 * If there is one non-busy active path at least, the path selector
1674 * will be able to select it. So we consider such a pg as not busy.
1675 */
1676 busy = 1;
1677 list_for_each_entry(pgpath, &pg->pgpaths, list)
1678 if (pgpath->is_active) {
1679 has_active = 1;
1680
1681 if (!__pgpath_busy(pgpath)) {
1682 busy = 0;
1683 break;
1684 }
1685 }
1686
1687 if (!has_active)
1688 /*
1689 * No active path in this pg, so this pg won't be used and
1690 * the current_pg will be changed at next mapping time.
1691 * We need to try mapping to determine it.
1692 */
1693 busy = 0;
1694
1695out:
1696 spin_unlock_irqrestore(&m->lock, flags);
1697
1698 return busy;
1699}
1700
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701/*-----------------------------------------------------------------
1702 * Module setup
1703 *---------------------------------------------------------------*/
1704static struct target_type multipath_target = {
1705 .name = "multipath",
Mike Snitzerbfebd1c2015-03-08 00:51:47 -05001706 .version = {1, 9, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707 .module = THIS_MODULE,
1708 .ctr = multipath_ctr,
1709 .dtr = multipath_dtr,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001710 .map_rq = multipath_map,
Mike Snitzere5863d92014-12-17 21:08:12 -05001711 .clone_and_map_rq = multipath_clone_and_map,
1712 .release_clone_rq = multipath_release_clone,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001713 .rq_end_io = multipath_end_io,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714 .presuspend = multipath_presuspend,
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001715 .postsuspend = multipath_postsuspend,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716 .resume = multipath_resume,
1717 .status = multipath_status,
1718 .message = multipath_message,
Milan Broz9af4aa32006-10-03 01:15:20 -07001719 .ioctl = multipath_ioctl,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001720 .iterate_devices = multipath_iterate_devices,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001721 .busy = multipath_busy,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001722};
1723
1724static int __init dm_multipath_init(void)
1725{
1726 int r;
1727
1728 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001729 _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001730 if (!_mpio_cache)
1731 return -ENOMEM;
1732
1733 r = dm_register_target(&multipath_target);
1734 if (r < 0) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001735 DMERR("register failed %d", r);
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001736 r = -EINVAL;
1737 goto bad_register_target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738 }
1739
Tejun Heo4d4d66a2011-01-13 19:59:57 +00001740 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001741 if (!kmultipathd) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001742 DMERR("failed to create workqueue kmpathd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001743 r = -ENOMEM;
1744 goto bad_alloc_kmultipathd;
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001745 }
1746
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001747 /*
1748 * A separate workqueue is used to handle the device handlers
1749 * to avoid overloading existing workqueue. Overloading the
1750 * old workqueue would also create a bottleneck in the
1751 * path of the storage hardware device activation.
1752 */
Tejun Heo4d4d66a2011-01-13 19:59:57 +00001753 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
1754 WQ_MEM_RECLAIM);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001755 if (!kmpath_handlerd) {
1756 DMERR("failed to create workqueue kmpath_handlerd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001757 r = -ENOMEM;
1758 goto bad_alloc_kmpath_handlerd;
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001759 }
1760
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001761 DMINFO("version %u.%u.%u loaded",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762 multipath_target.version[0], multipath_target.version[1],
1763 multipath_target.version[2]);
1764
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001765 return 0;
1766
1767bad_alloc_kmpath_handlerd:
1768 destroy_workqueue(kmultipathd);
1769bad_alloc_kmultipathd:
1770 dm_unregister_target(&multipath_target);
1771bad_register_target:
1772 kmem_cache_destroy(_mpio_cache);
1773
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774 return r;
1775}
1776
1777static void __exit dm_multipath_exit(void)
1778{
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001779 destroy_workqueue(kmpath_handlerd);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001780 destroy_workqueue(kmultipathd);
1781
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001782 dm_unregister_target(&multipath_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783 kmem_cache_destroy(_mpio_cache);
1784}
1785
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786module_init(dm_multipath_init);
1787module_exit(dm_multipath_exit);
1788
1789MODULE_DESCRIPTION(DM_NAME " multipath target");
1790MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
1791MODULE_LICENSE("GPL");