blob: 677ba223e2aef0217b0a9fb36735753d86a6df22 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
Mikulas Patocka586e80e2008-10-21 17:44:59 +01008#include <linux/device-mapper.h>
9
Mike Snitzerf4790822013-09-12 18:06:12 -040010#include "dm.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include "dm-path-selector.h"
Mike Andersonb15546f2007-10-19 22:48:02 +010012#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070013
Mike Snitzere5863d92014-12-17 21:08:12 -050014#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/ctype.h>
16#include <linux/init.h>
17#include <linux/mempool.h>
18#include <linux/module.h>
19#include <linux/pagemap.h>
20#include <linux/slab.h>
21#include <linux/time.h>
22#include <linux/workqueue.h>
Mikulas Patocka35991652012-06-03 00:29:58 +010023#include <linux/delay.h>
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070024#include <scsi/scsi_dh.h>
Arun Sharma600634972011-07-26 16:09:06 -070025#include <linux/atomic.h>
Mike Snitzer78ce23b2016-01-31 17:38:28 -050026#include <linux/blk-mq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Alasdair G Kergon72d94862006-06-26 00:27:35 -070028#define DM_MSG_PREFIX "multipath"
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000029#define DM_PG_INIT_DELAY_MSECS 2000
30#define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
Linus Torvalds1da177e2005-04-16 15:20:36 -070031
32/* Path properties */
33struct pgpath {
34 struct list_head list;
35
36 struct priority_group *pg; /* Owning PG */
37 unsigned fail_count; /* Cumulative failure count */
38
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -080039 struct dm_path path;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000040 struct delayed_work activate_path;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -050041
42 bool is_active:1; /* Path status */
Linus Torvalds1da177e2005-04-16 15:20:36 -070043};
44
45#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
46
47/*
48 * Paths are grouped into Priority Groups and numbered from 1 upwards.
49 * Each has a path selector which controls which path gets used.
50 */
51struct priority_group {
52 struct list_head list;
53
54 struct multipath *m; /* Owning multipath instance */
55 struct path_selector ps;
56
57 unsigned pg_num; /* Reference number */
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 unsigned nr_pgpaths; /* Number of paths in PG */
59 struct list_head pgpaths;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -050060
61 bool bypassed:1; /* Temporarily bypass this PG? */
Linus Torvalds1da177e2005-04-16 15:20:36 -070062};
63
64/* Multipath context */
65struct multipath {
66 struct list_head list;
67 struct dm_target *ti;
68
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070069 const char *hw_handler_name;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -070070 char *hw_handler_params;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000071
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010072 spinlock_t lock;
73
Linus Torvalds1da177e2005-04-16 15:20:36 -070074 unsigned nr_priority_groups;
75 struct list_head priority_groups;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000076
77 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
78
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -070079 unsigned pg_init_in_progress; /* Only one pg_init allowed at once */
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
81 unsigned nr_valid_paths; /* Total number of usable paths */
82 struct pgpath *current_pgpath;
83 struct priority_group *current_pg;
84 struct priority_group *next_pg; /* Switch to this PG if set */
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Mike Snitzerbe7d31c2016-02-10 13:02:21 -050086 bool queue_io:1; /* Must we queue all I/O? */
87 bool queue_if_no_path:1; /* Queue I/O if last path fails? */
88 bool saved_queue_if_no_path:1; /* Saved state during suspension */
89 bool retain_attached_hw_handler:1; /* If there's already a hw_handler present, don't change it. */
90 bool pg_init_disabled:1; /* pg_init is not currently allowed */
91 bool pg_init_required:1; /* pg_init needs calling? */
92 bool pg_init_delay_retry:1; /* Delay pg_init retry? */
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010093
Dave Wysochanskic9e45582007-10-19 22:47:53 +010094 unsigned pg_init_retries; /* Number of times to retry pg_init */
95 unsigned pg_init_count; /* Number of times pg_init called */
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000096 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
Linus Torvalds1da177e2005-04-16 15:20:36 -070097
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 struct work_struct trigger_event;
99
100 /*
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100101 * We must use a mempool of dm_mpath_io structs so that we
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 * can resubmit bios on error.
103 */
104 mempool_t *mpio_pool;
Mike Anderson6380f262009-12-10 23:52:21 +0000105
106 struct mutex work_mutex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107};
108
109/*
110 * Context information attached to each bio we process.
111 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100112struct dm_mpath_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 struct pgpath *pgpath;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100114 size_t nr_bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115};
116
117typedef int (*action_fn) (struct pgpath *pgpath);
118
Christoph Lametere18b8902006-12-06 20:33:20 -0800119static struct kmem_cache *_mpio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700121static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
David Howellsc4028952006-11-22 14:57:56 +0000122static void trigger_event(struct work_struct *work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700123static void activate_path(struct work_struct *work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124
125
126/*-----------------------------------------------
127 * Allocation routines
128 *-----------------------------------------------*/
129
130static struct pgpath *alloc_pgpath(void)
131{
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700132 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
Mike Anderson224cb3e2008-08-29 09:36:09 +0200134 if (pgpath) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500135 pgpath->is_active = true;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000136 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
Mike Anderson224cb3e2008-08-29 09:36:09 +0200137 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
139 return pgpath;
140}
141
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100142static void free_pgpath(struct pgpath *pgpath)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143{
144 kfree(pgpath);
145}
146
147static struct priority_group *alloc_priority_group(void)
148{
149 struct priority_group *pg;
150
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700151 pg = kzalloc(sizeof(*pg), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700153 if (pg)
154 INIT_LIST_HEAD(&pg->pgpaths);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155
156 return pg;
157}
158
159static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
160{
161 struct pgpath *pgpath, *tmp;
162
163 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
164 list_del(&pgpath->list);
165 dm_put_device(ti, pgpath->path.dev);
166 free_pgpath(pgpath);
167 }
168}
169
170static void free_priority_group(struct priority_group *pg,
171 struct dm_target *ti)
172{
173 struct path_selector *ps = &pg->ps;
174
175 if (ps->type) {
176 ps->type->destroy(ps);
177 dm_put_path_selector(ps->type);
178 }
179
180 free_pgpaths(&pg->pgpaths, ti);
181 kfree(pg);
182}
183
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500184static struct multipath *alloc_multipath(struct dm_target *ti, bool use_blk_mq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185{
186 struct multipath *m;
187
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700188 m = kzalloc(sizeof(*m), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 if (m) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 INIT_LIST_HEAD(&m->priority_groups);
191 spin_lock_init(&m->lock);
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500192 m->queue_io = true;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000193 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
David Howellsc4028952006-11-22 14:57:56 +0000194 INIT_WORK(&m->trigger_event, trigger_event);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000195 init_waitqueue_head(&m->pg_init_wait);
Mike Anderson6380f262009-12-10 23:52:21 +0000196 mutex_init(&m->work_mutex);
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500197
198 m->mpio_pool = NULL;
199 if (!use_blk_mq) {
200 unsigned min_ios = dm_get_reserved_rq_based_ios();
201
202 m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache);
203 if (!m->mpio_pool) {
204 kfree(m);
205 return NULL;
206 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 }
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500208
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700209 m->ti = ti;
210 ti->private = m;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 }
212
213 return m;
214}
215
216static void free_multipath(struct multipath *m)
217{
218 struct priority_group *pg, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219
220 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
221 list_del(&pg->list);
222 free_priority_group(pg, m->ti);
223 }
224
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700225 kfree(m->hw_handler_name);
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700226 kfree(m->hw_handler_params);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 mempool_destroy(m->mpio_pool);
228 kfree(m);
229}
230
Mike Snitzer2eff1922016-02-03 09:13:14 -0500231static struct dm_mpath_io *get_mpio(union map_info *info)
232{
233 return info->ptr;
234}
235
236static struct dm_mpath_io *set_mpio(struct multipath *m, union map_info *info)
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100237{
238 struct dm_mpath_io *mpio;
239
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500240 if (!m->mpio_pool) {
241 /* Use blk-mq pdu memory requested via per_io_data_size */
Mike Snitzer2eff1922016-02-03 09:13:14 -0500242 mpio = get_mpio(info);
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500243 memset(mpio, 0, sizeof(*mpio));
244 return mpio;
245 }
246
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100247 mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
248 if (!mpio)
Mike Snitzer2eff1922016-02-03 09:13:14 -0500249 return NULL;
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100250
251 memset(mpio, 0, sizeof(*mpio));
252 info->ptr = mpio;
253
Mike Snitzer2eff1922016-02-03 09:13:14 -0500254 return mpio;
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100255}
256
Mike Snitzer2eff1922016-02-03 09:13:14 -0500257static void clear_request_fn_mpio(struct multipath *m, union map_info *info)
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100258{
Mike Snitzer2eff1922016-02-03 09:13:14 -0500259 /* Only needed for non blk-mq (.request_fn) multipath */
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500260 if (m->mpio_pool) {
261 struct dm_mpath_io *mpio = info->ptr;
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100262
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500263 info->ptr = NULL;
264 mempool_free(mpio, m->mpio_pool);
265 }
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100266}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267
268/*-----------------------------------------------
269 * Path selection
270 *-----------------------------------------------*/
271
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100272static int __pg_init_all_paths(struct multipath *m)
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000273{
274 struct pgpath *pgpath;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000275 unsigned long pg_init_delay = 0;
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000276
Hannes Reinecke17f4ff42014-02-28 15:33:42 +0100277 if (m->pg_init_in_progress || m->pg_init_disabled)
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100278 return 0;
Hannes Reinecke17f4ff42014-02-28 15:33:42 +0100279
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000280 m->pg_init_count++;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500281 m->pg_init_required = false;
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100282
283 /* Check here to reset pg_init_required */
284 if (!m->current_pg)
285 return 0;
286
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000287 if (m->pg_init_delay_retry)
288 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
289 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000290 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
291 /* Skip failed paths */
292 if (!pgpath->is_active)
293 continue;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000294 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
295 pg_init_delay))
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000296 m->pg_init_in_progress++;
297 }
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100298 return m->pg_init_in_progress;
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000299}
300
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
302{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 m->current_pg = pgpath->pg;
304
305 /* Must we initialise the PG first, and queue I/O till it's ready? */
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700306 if (m->hw_handler_name) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500307 m->pg_init_required = true;
308 m->queue_io = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 } else {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500310 m->pg_init_required = false;
311 m->queue_io = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 }
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100313
314 m->pg_init_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315}
316
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100317static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
318 size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319{
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -0800320 struct dm_path *path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321
Mike Snitzer90a43232016-02-17 21:29:17 -0500322 path = pg->ps.type->select_path(&pg->ps, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 if (!path)
324 return -ENXIO;
325
326 m->current_pgpath = path_to_pgpath(path);
327
328 if (m->current_pg != pg)
329 __switch_pg(m, m->current_pgpath);
330
331 return 0;
332}
333
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100334static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335{
336 struct priority_group *pg;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500337 bool bypassed = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338
Benjamin Marzinski1f271972014-08-13 13:53:42 -0500339 if (!m->nr_valid_paths) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500340 m->queue_io = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 goto failed;
Benjamin Marzinski1f271972014-08-13 13:53:42 -0500342 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343
344 /* Were we instructed to switch PG? */
345 if (m->next_pg) {
346 pg = m->next_pg;
347 m->next_pg = NULL;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100348 if (!__choose_path_in_pg(m, pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 return;
350 }
351
352 /* Don't change PG until it has no remaining paths */
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100353 if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 return;
355
356 /*
357 * Loop through priority groups until we find a valid path.
358 * First time we skip PGs marked 'bypassed'.
Mike Christief220fd42012-06-03 00:29:45 +0100359 * Second time we only try the ones we skipped, but set
360 * pg_init_delay_retry so we do not hammer controllers.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 */
362 do {
363 list_for_each_entry(pg, &m->priority_groups, list) {
364 if (pg->bypassed == bypassed)
365 continue;
Mike Christief220fd42012-06-03 00:29:45 +0100366 if (!__choose_path_in_pg(m, pg, nr_bytes)) {
367 if (!bypassed)
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500368 m->pg_init_delay_retry = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 return;
Mike Christief220fd42012-06-03 00:29:45 +0100370 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 }
372 } while (bypassed--);
373
374failed:
375 m->current_pgpath = NULL;
376 m->current_pg = NULL;
377}
378
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800379/*
380 * Check whether bios must be queued in the device-mapper core rather
381 * than here in the target.
382 *
383 * m->lock must be held on entry.
384 *
385 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
386 * same value then we are not between multipath_presuspend()
387 * and multipath_resume() calls and we have no need to check
388 * for the DMF_NOFLUSH_SUSPENDING flag.
389 */
390static int __must_push_back(struct multipath *m)
391{
Hannes Reineckee8099172014-02-28 15:33:44 +0100392 return (m->queue_if_no_path ||
393 (m->queue_if_no_path != m->saved_queue_if_no_path &&
394 dm_noflush_suspending(m->ti)));
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800395}
396
Hannes Reinecke36fcffc2014-02-28 15:33:47 +0100397/*
398 * Map cloned requests
399 */
Mike Snitzere5863d92014-12-17 21:08:12 -0500400static int __multipath_map(struct dm_target *ti, struct request *clone,
401 union map_info *map_context,
402 struct request *rq, struct request **__clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403{
Mike Snitzer7943bd62016-02-02 21:53:15 -0500404 struct multipath *m = ti->private;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100405 int r = DM_MAPIO_REQUEUE;
Mike Snitzere5863d92014-12-17 21:08:12 -0500406 size_t nr_bytes = clone ? blk_rq_bytes(clone) : blk_rq_bytes(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 struct pgpath *pgpath;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100408 struct block_device *bdev;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100409 struct dm_mpath_io *mpio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600411 spin_lock_irq(&m->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412
413 /* Do we need to select a new pgpath? */
Mike Snitzer21136f82016-02-10 11:58:45 -0500414 if (!m->current_pgpath || !m->queue_io)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100415 __choose_pgpath(m, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416
417 pgpath = m->current_pgpath;
418
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100419 if (!pgpath) {
420 if (!__must_push_back(m))
421 r = -EIO; /* Failed */
422 goto out_unlock;
Mike Snitzer6afbc012014-07-08 11:55:09 -0400423 } else if (m->queue_io || m->pg_init_required) {
Hannes Reineckee3bde042014-02-28 15:33:46 +0100424 __pg_init_all_paths(m);
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100425 goto out_unlock;
426 }
Mike Snitzer6afbc012014-07-08 11:55:09 -0400427
Mike Snitzer2eff1922016-02-03 09:13:14 -0500428 mpio = set_mpio(m, map_context);
429 if (!mpio)
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100430 /* ENOMEM, requeue */
431 goto out_unlock;
432
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100433 mpio->pgpath = pgpath;
434 mpio->nr_bytes = nr_bytes;
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600435
436 bdev = pgpath->path.dev->bdev;
437
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600438 spin_unlock_irq(&m->lock);
439
Mike Snitzere5863d92014-12-17 21:08:12 -0500440 if (clone) {
Mike Snitzerc5248f72016-02-20 14:02:49 -0500441 /*
442 * Old request-based interface: allocated clone is passed in.
443 * Used by: .request_fn stacked on .request_fn path(s).
444 */
Mike Snitzere5863d92014-12-17 21:08:12 -0500445 clone->q = bdev_get_queue(bdev);
446 clone->rq_disk = bdev->bd_disk;
447 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
448 } else {
Mike Snitzereca7ee62016-02-20 13:45:38 -0500449 /*
450 * blk-mq request-based interface; used by both:
451 * .request_fn stacked on blk-mq path(s) and
452 * blk-mq stacked on blk-mq path(s).
453 */
Mike Snitzer78ce23b2016-01-31 17:38:28 -0500454 *__clone = blk_mq_alloc_request(bdev_get_queue(bdev),
455 rq_data_dir(rq), BLK_MQ_REQ_NOWAIT);
Mike Snitzer4c6dd532015-05-27 15:23:56 -0400456 if (IS_ERR(*__clone)) {
Mike Snitzere5863d92014-12-17 21:08:12 -0500457 /* ENOMEM, requeue */
Mike Snitzer2eff1922016-02-03 09:13:14 -0500458 clear_request_fn_mpio(m, map_context);
Mike Snitzere5863d92014-12-17 21:08:12 -0500459 return r;
Mike Snitzer4c6dd532015-05-27 15:23:56 -0400460 }
Mike Snitzere5863d92014-12-17 21:08:12 -0500461 (*__clone)->bio = (*__clone)->biotail = NULL;
462 (*__clone)->rq_disk = bdev->bd_disk;
463 (*__clone)->cmd_flags |= REQ_FAILFAST_TRANSPORT;
464 }
465
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100466 if (pgpath->pg->ps.type->start_io)
467 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
468 &pgpath->path,
469 nr_bytes);
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600470 return DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471
Hannes Reineckee3bde042014-02-28 15:33:46 +0100472out_unlock:
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600473 spin_unlock_irq(&m->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
475 return r;
476}
477
Mike Snitzere5863d92014-12-17 21:08:12 -0500478static int multipath_map(struct dm_target *ti, struct request *clone,
479 union map_info *map_context)
480{
481 return __multipath_map(ti, clone, map_context, NULL, NULL);
482}
483
484static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
485 union map_info *map_context,
486 struct request **clone)
487{
488 return __multipath_map(ti, NULL, map_context, rq, clone);
489}
490
491static void multipath_release_clone(struct request *clone)
492{
Mike Snitzer78ce23b2016-01-31 17:38:28 -0500493 blk_mq_free_request(clone);
Mike Snitzere5863d92014-12-17 21:08:12 -0500494}
495
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496/*
497 * If we run out of usable paths, should we queue I/O or error it?
498 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500499static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
500 bool save_old_value)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501{
502 unsigned long flags;
503
504 spin_lock_irqsave(&m->lock, flags);
505
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700506 if (save_old_value)
507 m->saved_queue_if_no_path = m->queue_if_no_path;
508 else
509 m->saved_queue_if_no_path = queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 m->queue_if_no_path = queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 spin_unlock_irqrestore(&m->lock, flags);
512
Hannes Reinecke63d832c2014-05-26 14:45:39 +0200513 if (!queue_if_no_path)
514 dm_table_run_md_queue_async(m->ti->table);
515
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 return 0;
517}
518
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519/*
520 * An event is triggered whenever a path is taken out of use.
521 * Includes path failure and PG bypass.
522 */
David Howellsc4028952006-11-22 14:57:56 +0000523static void trigger_event(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524{
David Howellsc4028952006-11-22 14:57:56 +0000525 struct multipath *m =
526 container_of(work, struct multipath, trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527
528 dm_table_event(m->ti->table);
529}
530
531/*-----------------------------------------------------------------
532 * Constructor/argument parsing:
533 * <#multipath feature args> [<arg>]*
534 * <#hw_handler args> [hw_handler [<arg>]*]
535 * <#priority groups>
536 * <initial priority group>
537 * [<selector> <#selector args> [<arg>]*
538 * <#paths> <#per-path selector args>
539 * [<path> [<arg>]* ]+ ]+
540 *---------------------------------------------------------------*/
Mike Snitzer498f0102011-08-02 12:32:04 +0100541static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 struct dm_target *ti)
543{
544 int r;
545 struct path_selector_type *pst;
546 unsigned ps_argc;
547
Mike Snitzer498f0102011-08-02 12:32:04 +0100548 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700549 {0, 1024, "invalid number of path selector args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 };
551
Mike Snitzer498f0102011-08-02 12:32:04 +0100552 pst = dm_get_path_selector(dm_shift_arg(as));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 if (!pst) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700554 ti->error = "unknown path selector type";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 return -EINVAL;
556 }
557
Mike Snitzer498f0102011-08-02 12:32:04 +0100558 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100559 if (r) {
560 dm_put_path_selector(pst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 return -EINVAL;
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100562 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
564 r = pst->create(&pg->ps, ps_argc, as->argv);
565 if (r) {
566 dm_put_path_selector(pst);
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700567 ti->error = "path selector constructor failed";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568 return r;
569 }
570
571 pg->ps.type = pst;
Mike Snitzer498f0102011-08-02 12:32:04 +0100572 dm_consume_args(as, ps_argc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573
574 return 0;
575}
576
Mike Snitzer498f0102011-08-02 12:32:04 +0100577static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 struct dm_target *ti)
579{
580 int r;
581 struct pgpath *p;
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700582 struct multipath *m = ti->private;
Mike Snitzera58a9352012-07-27 15:08:04 +0100583 struct request_queue *q = NULL;
584 const char *attached_handler_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
586 /* we need at least a path arg */
587 if (as->argc < 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700588 ti->error = "no device given";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100589 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 }
591
592 p = alloc_pgpath();
593 if (!p)
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100594 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
Mike Snitzer498f0102011-08-02 12:32:04 +0100596 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +0000597 &p->path.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700599 ti->error = "error getting device";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 goto bad;
601 }
602
Mike Snitzera58a9352012-07-27 15:08:04 +0100603 if (m->retain_attached_hw_handler || m->hw_handler_name)
604 q = bdev_get_queue(p->path.dev->bdev);
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100605
Mike Snitzera58a9352012-07-27 15:08:04 +0100606 if (m->retain_attached_hw_handler) {
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200607retain:
Mike Snitzera58a9352012-07-27 15:08:04 +0100608 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
609 if (attached_handler_name) {
610 /*
611 * Reset hw_handler_name to match the attached handler
612 * and clear any hw_handler_params associated with the
613 * ignored handler.
614 *
615 * NB. This modifies the table line to show the actual
616 * handler instead of the original table passed in.
617 */
618 kfree(m->hw_handler_name);
619 m->hw_handler_name = attached_handler_name;
620
621 kfree(m->hw_handler_params);
622 m->hw_handler_params = NULL;
623 }
624 }
625
626 if (m->hw_handler_name) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100627 r = scsi_dh_attach(q, m->hw_handler_name);
628 if (r == -EBUSY) {
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200629 char b[BDEVNAME_SIZE];
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100630
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200631 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
632 bdevname(p->path.dev->bdev, b));
633 goto retain;
634 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700635 if (r < 0) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100636 ti->error = "error attaching hardware handler";
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700637 dm_put_device(ti, p->path.dev);
638 goto bad;
639 }
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700640
641 if (m->hw_handler_params) {
642 r = scsi_dh_set_params(q, m->hw_handler_params);
643 if (r < 0) {
644 ti->error = "unable to set hardware "
645 "handler parameters";
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700646 dm_put_device(ti, p->path.dev);
647 goto bad;
648 }
649 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700650 }
651
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
653 if (r) {
654 dm_put_device(ti, p->path.dev);
655 goto bad;
656 }
657
658 return p;
659
660 bad:
661 free_pgpath(p);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100662 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663}
664
Mike Snitzer498f0102011-08-02 12:32:04 +0100665static struct priority_group *parse_priority_group(struct dm_arg_set *as,
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700666 struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667{
Mike Snitzer498f0102011-08-02 12:32:04 +0100668 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700669 {1, 1024, "invalid number of paths"},
670 {0, 1024, "invalid number of selector args"}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 };
672
673 int r;
Mike Snitzer498f0102011-08-02 12:32:04 +0100674 unsigned i, nr_selector_args, nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 struct priority_group *pg;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700676 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677
678 if (as->argc < 2) {
679 as->argc = 0;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100680 ti->error = "not enough priority group arguments";
681 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 }
683
684 pg = alloc_priority_group();
685 if (!pg) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700686 ti->error = "couldn't allocate priority group";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100687 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 }
689 pg->m = m;
690
691 r = parse_path_selector(as, pg, ti);
692 if (r)
693 goto bad;
694
695 /*
696 * read the paths
697 */
Mike Snitzer498f0102011-08-02 12:32:04 +0100698 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 if (r)
700 goto bad;
701
Mike Snitzer498f0102011-08-02 12:32:04 +0100702 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 if (r)
704 goto bad;
705
Mike Snitzer498f0102011-08-02 12:32:04 +0100706 nr_args = 1 + nr_selector_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 for (i = 0; i < pg->nr_pgpaths; i++) {
708 struct pgpath *pgpath;
Mike Snitzer498f0102011-08-02 12:32:04 +0100709 struct dm_arg_set path_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710
Mike Snitzer498f0102011-08-02 12:32:04 +0100711 if (as->argc < nr_args) {
Mikulas Patocka148acff2008-07-21 12:00:30 +0100712 ti->error = "not enough path parameters";
Alasdair G Kergon6bbf79a2010-08-12 04:13:49 +0100713 r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714 goto bad;
Mikulas Patocka148acff2008-07-21 12:00:30 +0100715 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716
Mike Snitzer498f0102011-08-02 12:32:04 +0100717 path_args.argc = nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 path_args.argv = as->argv;
719
720 pgpath = parse_path(&path_args, &pg->ps, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100721 if (IS_ERR(pgpath)) {
722 r = PTR_ERR(pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 goto bad;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100724 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725
726 pgpath->pg = pg;
727 list_add_tail(&pgpath->list, &pg->pgpaths);
Mike Snitzer498f0102011-08-02 12:32:04 +0100728 dm_consume_args(as, nr_args);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 }
730
731 return pg;
732
733 bad:
734 free_priority_group(pg, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100735 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736}
737
Mike Snitzer498f0102011-08-02 12:32:04 +0100738static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 unsigned hw_argc;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700741 int ret;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700742 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743
Mike Snitzer498f0102011-08-02 12:32:04 +0100744 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700745 {0, 1024, "invalid number of hardware handler args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 };
747
Mike Snitzer498f0102011-08-02 12:32:04 +0100748 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 return -EINVAL;
750
751 if (!hw_argc)
752 return 0;
753
Mike Snitzer498f0102011-08-02 12:32:04 +0100754 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
Chandra Seetharaman14e98c52008-11-13 23:39:06 +0000755
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700756 if (hw_argc > 1) {
757 char *p;
758 int i, j, len = 4;
759
760 for (i = 0; i <= hw_argc - 2; i++)
761 len += strlen(as->argv[i]) + 1;
762 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
763 if (!p) {
764 ti->error = "memory allocation failed";
765 ret = -ENOMEM;
766 goto fail;
767 }
768 j = sprintf(p, "%d", hw_argc - 1);
769 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
770 j = sprintf(p, "%s", as->argv[i]);
771 }
Mike Snitzer498f0102011-08-02 12:32:04 +0100772 dm_consume_args(as, hw_argc - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773
774 return 0;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700775fail:
776 kfree(m->hw_handler_name);
777 m->hw_handler_name = NULL;
778 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779}
780
Mike Snitzer498f0102011-08-02 12:32:04 +0100781static int parse_features(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782{
783 int r;
784 unsigned argc;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700785 struct dm_target *ti = m->ti;
Mike Snitzer498f0102011-08-02 12:32:04 +0100786 const char *arg_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787
Mike Snitzer498f0102011-08-02 12:32:04 +0100788 static struct dm_arg _args[] = {
Mike Snitzera58a9352012-07-27 15:08:04 +0100789 {0, 6, "invalid number of feature args"},
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100790 {1, 50, "pg_init_retries must be between 1 and 50"},
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000791 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 };
793
Mike Snitzer498f0102011-08-02 12:32:04 +0100794 r = dm_read_arg_group(_args, as, &argc, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 if (r)
796 return -EINVAL;
797
798 if (!argc)
799 return 0;
800
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100801 do {
Mike Snitzer498f0102011-08-02 12:32:04 +0100802 arg_name = dm_shift_arg(as);
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100803 argc--;
804
Mike Snitzer498f0102011-08-02 12:32:04 +0100805 if (!strcasecmp(arg_name, "queue_if_no_path")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500806 r = queue_if_no_path(m, true, false);
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100807 continue;
808 }
809
Mike Snitzera58a9352012-07-27 15:08:04 +0100810 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500811 m->retain_attached_hw_handler = true;
Mike Snitzera58a9352012-07-27 15:08:04 +0100812 continue;
813 }
814
Mike Snitzer498f0102011-08-02 12:32:04 +0100815 if (!strcasecmp(arg_name, "pg_init_retries") &&
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100816 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +0100817 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100818 argc--;
819 continue;
820 }
821
Mike Snitzer498f0102011-08-02 12:32:04 +0100822 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000823 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +0100824 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000825 argc--;
826 continue;
827 }
828
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 ti->error = "Unrecognised multipath feature request";
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100830 r = -EINVAL;
831 } while (argc && !r);
832
833 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834}
835
836static int multipath_ctr(struct dm_target *ti, unsigned int argc,
837 char **argv)
838{
Mike Snitzer498f0102011-08-02 12:32:04 +0100839 /* target arguments */
840 static struct dm_arg _args[] = {
Mike Snitzera490a072011-03-24 13:54:33 +0000841 {0, 1024, "invalid number of priority groups"},
842 {0, 1024, "invalid initial priority group number"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843 };
844
845 int r;
846 struct multipath *m;
Mike Snitzer498f0102011-08-02 12:32:04 +0100847 struct dm_arg_set as;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 unsigned pg_count = 0;
849 unsigned next_pg_num;
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500850 bool use_blk_mq = dm_use_blk_mq(dm_table_get_md(ti->table));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851
852 as.argc = argc;
853 as.argv = argv;
854
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500855 m = alloc_multipath(ti, use_blk_mq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 if (!m) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700857 ti->error = "can't allocate multipath";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 return -EINVAL;
859 }
860
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700861 r = parse_features(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862 if (r)
863 goto bad;
864
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700865 r = parse_hw_handler(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866 if (r)
867 goto bad;
868
Mike Snitzer498f0102011-08-02 12:32:04 +0100869 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 if (r)
871 goto bad;
872
Mike Snitzer498f0102011-08-02 12:32:04 +0100873 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 if (r)
875 goto bad;
876
Mike Snitzera490a072011-03-24 13:54:33 +0000877 if ((!m->nr_priority_groups && next_pg_num) ||
878 (m->nr_priority_groups && !next_pg_num)) {
879 ti->error = "invalid initial priority group";
880 r = -EINVAL;
881 goto bad;
882 }
883
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 /* parse the priority groups */
885 while (as.argc) {
886 struct priority_group *pg;
887
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700888 pg = parse_priority_group(&as, m);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100889 if (IS_ERR(pg)) {
890 r = PTR_ERR(pg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 goto bad;
892 }
893
894 m->nr_valid_paths += pg->nr_pgpaths;
895 list_add_tail(&pg->list, &m->priority_groups);
896 pg_count++;
897 pg->pg_num = pg_count;
898 if (!--next_pg_num)
899 m->next_pg = pg;
900 }
901
902 if (pg_count != m->nr_priority_groups) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700903 ti->error = "priority group count mismatch";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904 r = -EINVAL;
905 goto bad;
906 }
907
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +0000908 ti->num_flush_bios = 1;
909 ti->num_discard_bios = 1;
Mike Snitzer042bcef2013-05-10 14:37:16 +0100910 ti->num_write_same_bios = 1;
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500911 if (use_blk_mq)
912 ti->per_io_data_size = sizeof(struct dm_mpath_io);
Mikulas Patocka86279212009-06-22 10:12:24 +0100913
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 return 0;
915
916 bad:
917 free_multipath(m);
918 return r;
919}
920
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000921static void multipath_wait_for_pg_init_completion(struct multipath *m)
922{
923 DECLARE_WAITQUEUE(wait, current);
924 unsigned long flags;
925
926 add_wait_queue(&m->pg_init_wait, &wait);
927
928 while (1) {
929 set_current_state(TASK_UNINTERRUPTIBLE);
930
931 spin_lock_irqsave(&m->lock, flags);
932 if (!m->pg_init_in_progress) {
933 spin_unlock_irqrestore(&m->lock, flags);
934 break;
935 }
936 spin_unlock_irqrestore(&m->lock, flags);
937
938 io_schedule();
939 }
940 set_current_state(TASK_RUNNING);
941
942 remove_wait_queue(&m->pg_init_wait, &wait);
943}
944
945static void flush_multipath_work(struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946{
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000947 unsigned long flags;
948
949 spin_lock_irqsave(&m->lock, flags);
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500950 m->pg_init_disabled = true;
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000951 spin_unlock_irqrestore(&m->lock, flags);
952
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700953 flush_workqueue(kmpath_handlerd);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000954 multipath_wait_for_pg_init_completion(m);
Alasdair G Kergona044d012005-07-12 15:53:02 -0700955 flush_workqueue(kmultipathd);
Tejun Heo43829732012-08-20 14:51:24 -0700956 flush_work(&m->trigger_event);
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000957
958 spin_lock_irqsave(&m->lock, flags);
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500959 m->pg_init_disabled = false;
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +0000960 spin_unlock_irqrestore(&m->lock, flags);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +0000961}
962
963static void multipath_dtr(struct dm_target *ti)
964{
965 struct multipath *m = ti->private;
966
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000967 flush_multipath_work(m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 free_multipath(m);
969}
970
971/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 * Take a path out of use.
973 */
974static int fail_path(struct pgpath *pgpath)
975{
976 unsigned long flags;
977 struct multipath *m = pgpath->pg->m;
978
979 spin_lock_irqsave(&m->lock, flags);
980
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100981 if (!pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 goto out;
983
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700984 DMWARN("Failing path %s.", pgpath->path.dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985
986 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500987 pgpath->is_active = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 pgpath->fail_count++;
989
990 m->nr_valid_paths--;
991
992 if (pgpath == m->current_pgpath)
993 m->current_pgpath = NULL;
994
Mike Andersonb15546f2007-10-19 22:48:02 +0100995 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
996 pgpath->path.dev->name, m->nr_valid_paths);
997
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +0000998 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999
1000out:
1001 spin_unlock_irqrestore(&m->lock, flags);
1002
1003 return 0;
1004}
1005
1006/*
1007 * Reinstate a previously-failed path
1008 */
1009static int reinstate_path(struct pgpath *pgpath)
1010{
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001011 int r = 0, run_queue = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 unsigned long flags;
1013 struct multipath *m = pgpath->pg->m;
1014
1015 spin_lock_irqsave(&m->lock, flags);
1016
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001017 if (pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 goto out;
1019
Mike Snitzerec31f3f2016-02-20 12:49:43 -05001020 DMWARN("Reinstating path %s.", pgpath->path.dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021
1022 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1023 if (r)
1024 goto out;
1025
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001026 pgpath->is_active = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027
Hannes Reineckee8099172014-02-28 15:33:44 +01001028 if (!m->nr_valid_paths++) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001029 m->current_pgpath = NULL;
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001030 run_queue = 1;
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001031 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001032 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001033 m->pg_init_in_progress++;
1034 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035
Mike Andersonb15546f2007-10-19 22:48:02 +01001036 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1037 pgpath->path.dev->name, m->nr_valid_paths);
1038
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001039 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040
1041out:
1042 spin_unlock_irqrestore(&m->lock, flags);
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001043 if (run_queue)
1044 dm_table_run_md_queue_async(m->ti->table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045
1046 return r;
1047}
1048
1049/*
1050 * Fail or reinstate all paths that match the provided struct dm_dev.
1051 */
1052static int action_dev(struct multipath *m, struct dm_dev *dev,
1053 action_fn action)
1054{
Mike Snitzer19040c02011-03-24 13:54:31 +00001055 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 struct pgpath *pgpath;
1057 struct priority_group *pg;
1058
1059 list_for_each_entry(pg, &m->priority_groups, list) {
1060 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1061 if (pgpath->path.dev == dev)
1062 r = action(pgpath);
1063 }
1064 }
1065
1066 return r;
1067}
1068
1069/*
1070 * Temporarily try to avoid having to use the specified PG
1071 */
1072static void bypass_pg(struct multipath *m, struct priority_group *pg,
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001073 bool bypassed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074{
1075 unsigned long flags;
1076
1077 spin_lock_irqsave(&m->lock, flags);
1078
1079 pg->bypassed = bypassed;
1080 m->current_pgpath = NULL;
1081 m->current_pg = NULL;
1082
1083 spin_unlock_irqrestore(&m->lock, flags);
1084
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001085 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086}
1087
1088/*
1089 * Switch to using the specified PG from the next I/O that gets mapped
1090 */
1091static int switch_pg_num(struct multipath *m, const char *pgstr)
1092{
1093 struct priority_group *pg;
1094 unsigned pgnum;
1095 unsigned long flags;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001096 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001098 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 (pgnum > m->nr_priority_groups)) {
1100 DMWARN("invalid PG number supplied to switch_pg_num");
1101 return -EINVAL;
1102 }
1103
1104 spin_lock_irqsave(&m->lock, flags);
1105 list_for_each_entry(pg, &m->priority_groups, list) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001106 pg->bypassed = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107 if (--pgnum)
1108 continue;
1109
1110 m->current_pgpath = NULL;
1111 m->current_pg = NULL;
1112 m->next_pg = pg;
1113 }
1114 spin_unlock_irqrestore(&m->lock, flags);
1115
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001116 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117 return 0;
1118}
1119
1120/*
1121 * Set/clear bypassed status of a PG.
1122 * PGs are numbered upwards from 1 in the order they were declared.
1123 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001124static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125{
1126 struct priority_group *pg;
1127 unsigned pgnum;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001128 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001130 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131 (pgnum > m->nr_priority_groups)) {
1132 DMWARN("invalid PG number supplied to bypass_pg");
1133 return -EINVAL;
1134 }
1135
1136 list_for_each_entry(pg, &m->priority_groups, list) {
1137 if (!--pgnum)
1138 break;
1139 }
1140
1141 bypass_pg(m, pg, bypassed);
1142 return 0;
1143}
1144
1145/*
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001146 * Should we retry pg_init immediately?
1147 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001148static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001149{
1150 unsigned long flags;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001151 bool limit_reached = false;
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001152
1153 spin_lock_irqsave(&m->lock, flags);
1154
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +00001155 if (m->pg_init_count <= m->pg_init_retries && !m->pg_init_disabled)
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001156 m->pg_init_required = true;
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001157 else
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001158 limit_reached = true;
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001159
1160 spin_unlock_irqrestore(&m->lock, flags);
1161
1162 return limit_reached;
1163}
1164
Chandra Seetharaman3ae31f62009-10-21 09:22:46 -07001165static void pg_init_done(void *data, int errors)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001166{
Moger, Babu83c0d5d2010-03-06 02:29:45 +00001167 struct pgpath *pgpath = data;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001168 struct priority_group *pg = pgpath->pg;
1169 struct multipath *m = pg->m;
1170 unsigned long flags;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001171 bool delay_retry = false;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001172
1173 /* device or driver problems */
1174 switch (errors) {
1175 case SCSI_DH_OK:
1176 break;
1177 case SCSI_DH_NOSYS:
1178 if (!m->hw_handler_name) {
1179 errors = 0;
1180 break;
1181 }
Moger, Babuf7b934c2010-03-06 02:29:49 +00001182 DMERR("Could not failover the device: Handler scsi_dh_%s "
1183 "Error %d.", m->hw_handler_name, errors);
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001184 /*
1185 * Fail path for now, so we do not ping pong
1186 */
1187 fail_path(pgpath);
1188 break;
1189 case SCSI_DH_DEV_TEMP_BUSY:
1190 /*
1191 * Probably doing something like FW upgrade on the
1192 * controller so try the other pg.
1193 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001194 bypass_pg(m, pg, true);
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001195 break;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001196 case SCSI_DH_RETRY:
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001197 /* Wait before retrying. */
1198 delay_retry = 1;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001199 case SCSI_DH_IMM_RETRY:
1200 case SCSI_DH_RES_TEMP_UNAVAIL:
1201 if (pg_init_limit_reached(m, pgpath))
1202 fail_path(pgpath);
1203 errors = 0;
1204 break;
Mike Snitzerec31f3f2016-02-20 12:49:43 -05001205 case SCSI_DH_DEV_OFFLINED:
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001206 default:
1207 /*
1208 * We probably do not want to fail the path for a device
1209 * error, but this is what the old dm did. In future
1210 * patches we can do more advanced handling.
1211 */
1212 fail_path(pgpath);
1213 }
1214
1215 spin_lock_irqsave(&m->lock, flags);
1216 if (errors) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001217 if (pgpath == m->current_pgpath) {
1218 DMERR("Could not failover device. Error %d.", errors);
1219 m->current_pgpath = NULL;
1220 m->current_pg = NULL;
1221 }
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001222 } else if (!m->pg_init_required)
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001223 pg->bypassed = false;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001224
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001225 if (--m->pg_init_in_progress)
1226 /* Activations of other paths are still on going */
1227 goto out;
1228
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001229 if (m->pg_init_required) {
1230 m->pg_init_delay_retry = delay_retry;
1231 if (__pg_init_all_paths(m))
1232 goto out;
1233 }
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001234 m->queue_io = false;
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001235
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001236 /*
1237 * Wake up any thread waiting to suspend.
1238 */
1239 wake_up(&m->pg_init_wait);
1240
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001241out:
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001242 spin_unlock_irqrestore(&m->lock, flags);
1243}
1244
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001245static void activate_path(struct work_struct *work)
1246{
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001247 struct pgpath *pgpath =
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001248 container_of(work, struct pgpath, activate_path.work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001249
Hannes Reinecke3a017502014-02-28 15:33:49 +01001250 if (pgpath->is_active)
1251 scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
1252 pg_init_done, pgpath);
1253 else
1254 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001255}
1256
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001257static int noretry_error(int error)
1258{
1259 switch (error) {
1260 case -EOPNOTSUPP:
1261 case -EREMOTEIO:
1262 case -EILSEQ:
1263 case -ENODATA:
Jun'ichi Nomuracc9d3c32013-09-13 14:54:30 +09001264 case -ENOSPC:
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001265 return 1;
1266 }
1267
1268 /* Anything else could be a path failure, so should be retried */
1269 return 0;
1270}
1271
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272/*
1273 * end_io handling
1274 */
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001275static int do_end_io(struct multipath *m, struct request *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001276 int error, struct dm_mpath_io *mpio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277{
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001278 /*
1279 * We don't queue any clone request inside the multipath target
1280 * during end I/O handling, since those clone requests don't have
1281 * bio clones. If we queue them inside the multipath target,
1282 * we need to make bio clones, that requires memory allocation.
1283 * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
1284 * don't have bio clones.)
1285 * Instead of queueing the clone request here, we queue the original
1286 * request into dm core, which will remake a clone request and
1287 * clone bios for it and resubmit it later.
1288 */
1289 int r = DM_ENDIO_REQUEUE;
Stefan Bader640eb3b2005-11-21 21:32:35 -08001290 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001292 if (!error && !clone->errors)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293 return 0; /* I/O complete */
1294
Mike Snitzer7eee4ae2014-06-02 15:50:06 -04001295 if (noretry_error(error))
Mike Snitzer959eb4e2010-08-12 04:14:32 +01001296 return error;
1297
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001298 if (mpio->pgpath)
1299 fail_path(mpio->pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300
Stefan Bader640eb3b2005-11-21 21:32:35 -08001301 spin_lock_irqsave(&m->lock, flags);
Hannes Reinecke751b2a72011-01-18 10:13:12 +01001302 if (!m->nr_valid_paths) {
1303 if (!m->queue_if_no_path) {
1304 if (!__must_push_back(m))
1305 r = -EIO;
1306 } else {
1307 if (error == -EBADE)
1308 r = error;
1309 }
1310 }
Stefan Bader640eb3b2005-11-21 21:32:35 -08001311 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001313 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314}
1315
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001316static int multipath_end_io(struct dm_target *ti, struct request *clone,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 int error, union map_info *map_context)
1318{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001319 struct multipath *m = ti->private;
Mike Snitzer2eff1922016-02-03 09:13:14 -05001320 struct dm_mpath_io *mpio = get_mpio(map_context);
Wei Yongjuna71a2612012-10-12 16:59:42 +01001321 struct pgpath *pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322 struct path_selector *ps;
1323 int r;
1324
Jun'ichi Nomura466891f2012-03-28 18:41:25 +01001325 BUG_ON(!mpio);
1326
Mike Snitzer2eff1922016-02-03 09:13:14 -05001327 r = do_end_io(m, clone, error, mpio);
Wei Yongjuna71a2612012-10-12 16:59:42 +01001328 pgpath = mpio->pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001329 if (pgpath) {
1330 ps = &pgpath->pg->ps;
1331 if (ps->type->end_io)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001332 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 }
Mike Snitzer2eff1922016-02-03 09:13:14 -05001334 clear_request_fn_mpio(m, map_context);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335
1336 return r;
1337}
1338
1339/*
1340 * Suspend can't complete until all the I/O is processed so if
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001341 * the last path fails we must error any remaining I/O.
1342 * Note that if the freeze_bdev fails while suspending, the
1343 * queue_if_no_path state is lost - userspace should reset it.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 */
1345static void multipath_presuspend(struct dm_target *ti)
1346{
Mike Snitzer7943bd62016-02-02 21:53:15 -05001347 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001349 queue_if_no_path(m, false, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350}
1351
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001352static void multipath_postsuspend(struct dm_target *ti)
1353{
Mike Anderson6380f262009-12-10 23:52:21 +00001354 struct multipath *m = ti->private;
1355
1356 mutex_lock(&m->work_mutex);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001357 flush_multipath_work(m);
Mike Anderson6380f262009-12-10 23:52:21 +00001358 mutex_unlock(&m->work_mutex);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001359}
1360
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001361/*
1362 * Restore the queue_if_no_path setting.
1363 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364static void multipath_resume(struct dm_target *ti)
1365{
Mike Snitzer7943bd62016-02-02 21:53:15 -05001366 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 unsigned long flags;
1368
1369 spin_lock_irqsave(&m->lock, flags);
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001370 m->queue_if_no_path = m->saved_queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 spin_unlock_irqrestore(&m->lock, flags);
1372}
1373
1374/*
1375 * Info output has the following format:
1376 * num_multipath_feature_args [multipath_feature_args]*
1377 * num_handler_status_args [handler_status_args]*
1378 * num_groups init_group_number
1379 * [A|D|E num_ps_status_args [ps_status_args]*
1380 * num_paths num_selector_args
1381 * [path_dev A|F fail_count [selector_args]* ]+ ]+
1382 *
1383 * Table output has the following format (identical to the constructor string):
1384 * num_feature_args [features_args]*
1385 * num_handler_args hw_handler [hw_handler_args]*
1386 * num_groups init_group_number
1387 * [priority selector-name num_ps_args [ps_args]*
1388 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1389 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00001390static void multipath_status(struct dm_target *ti, status_type_t type,
1391 unsigned status_flags, char *result, unsigned maxlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392{
1393 int sz = 0;
1394 unsigned long flags;
Mike Snitzer7943bd62016-02-02 21:53:15 -05001395 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 struct priority_group *pg;
1397 struct pgpath *p;
1398 unsigned pg_num;
1399 char state;
1400
1401 spin_lock_irqsave(&m->lock, flags);
1402
1403 /* Features */
1404 if (type == STATUSTYPE_INFO)
Hannes Reineckee8099172014-02-28 15:33:44 +01001405 DMEMIT("2 %u %u ", m->queue_io, m->pg_init_count);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001406 else {
1407 DMEMIT("%u ", m->queue_if_no_path +
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001408 (m->pg_init_retries > 0) * 2 +
Mike Snitzera58a9352012-07-27 15:08:04 +01001409 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1410 m->retain_attached_hw_handler);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001411 if (m->queue_if_no_path)
1412 DMEMIT("queue_if_no_path ");
1413 if (m->pg_init_retries)
1414 DMEMIT("pg_init_retries %u ", m->pg_init_retries);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001415 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1416 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
Mike Snitzera58a9352012-07-27 15:08:04 +01001417 if (m->retain_attached_hw_handler)
1418 DMEMIT("retain_attached_hw_handler ");
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001419 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001420
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001421 if (!m->hw_handler_name || type == STATUSTYPE_INFO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422 DMEMIT("0 ");
1423 else
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001424 DMEMIT("1 %s ", m->hw_handler_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425
1426 DMEMIT("%u ", m->nr_priority_groups);
1427
1428 if (m->next_pg)
1429 pg_num = m->next_pg->pg_num;
1430 else if (m->current_pg)
1431 pg_num = m->current_pg->pg_num;
1432 else
Mike Snitzera490a072011-03-24 13:54:33 +00001433 pg_num = (m->nr_priority_groups ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434
1435 DMEMIT("%u ", pg_num);
1436
1437 switch (type) {
1438 case STATUSTYPE_INFO:
1439 list_for_each_entry(pg, &m->priority_groups, list) {
1440 if (pg->bypassed)
1441 state = 'D'; /* Disabled */
1442 else if (pg == m->current_pg)
1443 state = 'A'; /* Currently Active */
1444 else
1445 state = 'E'; /* Enabled */
1446
1447 DMEMIT("%c ", state);
1448
1449 if (pg->ps.type->status)
1450 sz += pg->ps.type->status(&pg->ps, NULL, type,
1451 result + sz,
1452 maxlen - sz);
1453 else
1454 DMEMIT("0 ");
1455
1456 DMEMIT("%u %u ", pg->nr_pgpaths,
1457 pg->ps.type->info_args);
1458
1459 list_for_each_entry(p, &pg->pgpaths, list) {
1460 DMEMIT("%s %s %u ", p->path.dev->name,
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001461 p->is_active ? "A" : "F",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462 p->fail_count);
1463 if (pg->ps.type->status)
1464 sz += pg->ps.type->status(&pg->ps,
1465 &p->path, type, result + sz,
1466 maxlen - sz);
1467 }
1468 }
1469 break;
1470
1471 case STATUSTYPE_TABLE:
1472 list_for_each_entry(pg, &m->priority_groups, list) {
1473 DMEMIT("%s ", pg->ps.type->name);
1474
1475 if (pg->ps.type->status)
1476 sz += pg->ps.type->status(&pg->ps, NULL, type,
1477 result + sz,
1478 maxlen - sz);
1479 else
1480 DMEMIT("0 ");
1481
1482 DMEMIT("%u %u ", pg->nr_pgpaths,
1483 pg->ps.type->table_args);
1484
1485 list_for_each_entry(p, &pg->pgpaths, list) {
1486 DMEMIT("%s ", p->path.dev->name);
1487 if (pg->ps.type->status)
1488 sz += pg->ps.type->status(&pg->ps,
1489 &p->path, type, result + sz,
1490 maxlen - sz);
1491 }
1492 }
1493 break;
1494 }
1495
1496 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497}
1498
1499static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1500{
Mike Anderson6380f262009-12-10 23:52:21 +00001501 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 struct dm_dev *dev;
Mike Snitzer7943bd62016-02-02 21:53:15 -05001503 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 action_fn action;
1505
Mike Anderson6380f262009-12-10 23:52:21 +00001506 mutex_lock(&m->work_mutex);
1507
Kiyoshi Uedac2f3d242009-12-10 23:52:27 +00001508 if (dm_suspended(ti)) {
1509 r = -EBUSY;
1510 goto out;
1511 }
1512
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513 if (argc == 1) {
Mike Snitzer498f0102011-08-02 12:32:04 +01001514 if (!strcasecmp(argv[0], "queue_if_no_path")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001515 r = queue_if_no_path(m, true, false);
Mike Anderson6380f262009-12-10 23:52:21 +00001516 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001517 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001518 r = queue_if_no_path(m, false, false);
Mike Anderson6380f262009-12-10 23:52:21 +00001519 goto out;
1520 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 }
1522
Mike Anderson6380f262009-12-10 23:52:21 +00001523 if (argc != 2) {
Jose Castilloa356e422014-01-29 17:52:45 +01001524 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
Mike Anderson6380f262009-12-10 23:52:21 +00001525 goto out;
1526 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527
Mike Snitzer498f0102011-08-02 12:32:04 +01001528 if (!strcasecmp(argv[0], "disable_group")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001529 r = bypass_pg_num(m, argv[1], true);
Mike Anderson6380f262009-12-10 23:52:21 +00001530 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001531 } else if (!strcasecmp(argv[0], "enable_group")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001532 r = bypass_pg_num(m, argv[1], false);
Mike Anderson6380f262009-12-10 23:52:21 +00001533 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001534 } else if (!strcasecmp(argv[0], "switch_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001535 r = switch_pg_num(m, argv[1]);
1536 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001537 } else if (!strcasecmp(argv[0], "reinstate_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 action = reinstate_path;
Mike Snitzer498f0102011-08-02 12:32:04 +01001539 else if (!strcasecmp(argv[0], "fail_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540 action = fail_path;
Mike Anderson6380f262009-12-10 23:52:21 +00001541 else {
Jose Castilloa356e422014-01-29 17:52:45 +01001542 DMWARN("Unrecognised multipath message received: %s", argv[0]);
Mike Anderson6380f262009-12-10 23:52:21 +00001543 goto out;
1544 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +00001546 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001548 DMWARN("message: error getting device %s",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549 argv[1]);
Mike Anderson6380f262009-12-10 23:52:21 +00001550 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 }
1552
1553 r = action_dev(m, dev, action);
1554
1555 dm_put_device(ti, dev);
1556
Mike Anderson6380f262009-12-10 23:52:21 +00001557out:
1558 mutex_unlock(&m->work_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560}
1561
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001562static int multipath_prepare_ioctl(struct dm_target *ti,
1563 struct block_device **bdev, fmode_t *mode)
Milan Broz9af4aa32006-10-03 01:15:20 -07001564{
Mikulas Patocka35991652012-06-03 00:29:58 +01001565 struct multipath *m = ti->private;
Milan Broz9af4aa32006-10-03 01:15:20 -07001566 unsigned long flags;
Mikulas Patocka35991652012-06-03 00:29:58 +01001567 int r;
1568
Milan Broz9af4aa32006-10-03 01:15:20 -07001569 spin_lock_irqsave(&m->lock, flags);
1570
1571 if (!m->current_pgpath)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001572 __choose_pgpath(m, 0);
Milan Broz9af4aa32006-10-03 01:15:20 -07001573
Junichi Nomura43e43c92015-11-17 09:36:56 +00001574 if (m->current_pgpath) {
1575 if (!m->queue_io) {
1576 *bdev = m->current_pgpath->path.dev->bdev;
1577 *mode = m->current_pgpath->path.dev->mode;
1578 r = 0;
1579 } else {
1580 /* pg_init has not started or completed */
1581 r = -ENOTCONN;
1582 }
1583 } else {
1584 /* No path is available */
1585 if (m->queue_if_no_path)
1586 r = -ENOTCONN;
1587 else
1588 r = -EIO;
Milan Broze90dae12006-10-03 01:15:22 -07001589 }
Milan Broz9af4aa32006-10-03 01:15:20 -07001590
Milan Broz9af4aa32006-10-03 01:15:20 -07001591 spin_unlock_irqrestore(&m->lock, flags);
1592
Junichi Nomura5bbbfdf2015-11-17 09:39:26 +00001593 if (r == -ENOTCONN) {
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001594 spin_lock_irqsave(&m->lock, flags);
1595 if (!m->current_pg) {
1596 /* Path status changed, redo selection */
1597 __choose_pgpath(m, 0);
1598 }
1599 if (m->pg_init_required)
1600 __pg_init_all_paths(m);
Mike Snitzer4cdd2ad2014-05-13 13:49:39 -04001601 spin_unlock_irqrestore(&m->lock, flags);
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001602 dm_table_run_md_queue_async(m->ti->table);
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001603 }
Mikulas Patocka35991652012-06-03 00:29:58 +01001604
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001605 /*
1606 * Only pass ioctls through if the device sizes match exactly.
1607 */
1608 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1609 return 1;
1610 return r;
Milan Broz9af4aa32006-10-03 01:15:20 -07001611}
1612
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001613static int multipath_iterate_devices(struct dm_target *ti,
1614 iterate_devices_callout_fn fn, void *data)
1615{
1616 struct multipath *m = ti->private;
1617 struct priority_group *pg;
1618 struct pgpath *p;
1619 int ret = 0;
1620
1621 list_for_each_entry(pg, &m->priority_groups, list) {
1622 list_for_each_entry(p, &pg->pgpaths, list) {
Mike Snitzer5dea2712009-07-23 20:30:42 +01001623 ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001624 if (ret)
1625 goto out;
1626 }
1627 }
1628
1629out:
1630 return ret;
1631}
1632
Mike Snitzer9f54cec2016-02-11 21:42:28 -05001633static int pgpath_busy(struct pgpath *pgpath)
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001634{
1635 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1636
Mike Snitzer52b09912015-02-23 16:36:41 -05001637 return blk_lld_busy(q);
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001638}
1639
1640/*
1641 * We return "busy", only when we can map I/Os but underlying devices
1642 * are busy (so even if we map I/Os now, the I/Os will wait on
1643 * the underlying queue).
1644 * In other words, if we want to kill I/Os or queue them inside us
1645 * due to map unavailability, we don't return "busy". Otherwise,
1646 * dm core won't give us the I/Os and we can't do what we want.
1647 */
1648static int multipath_busy(struct dm_target *ti)
1649{
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001650 bool busy = false, has_active = false;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001651 struct multipath *m = ti->private;
1652 struct priority_group *pg;
1653 struct pgpath *pgpath;
1654 unsigned long flags;
1655
1656 spin_lock_irqsave(&m->lock, flags);
1657
Jun'ichi Nomura7a7a3b42014-07-08 00:55:14 +00001658 /* pg_init in progress or no paths available */
1659 if (m->pg_init_in_progress ||
1660 (!m->nr_valid_paths && m->queue_if_no_path)) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001661 busy = true;
Hannes Reineckeb63349a2013-10-01 11:49:56 +02001662 goto out;
1663 }
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001664 /* Guess which priority_group will be used at next mapping time */
1665 if (unlikely(!m->current_pgpath && m->next_pg))
1666 pg = m->next_pg;
1667 else if (likely(m->current_pg))
1668 pg = m->current_pg;
1669 else
1670 /*
1671 * We don't know which pg will be used at next mapping time.
1672 * We don't call __choose_pgpath() here to avoid to trigger
1673 * pg_init just by busy checking.
1674 * So we don't know whether underlying devices we will be using
1675 * at next mapping time are busy or not. Just try mapping.
1676 */
1677 goto out;
1678
1679 /*
1680 * If there is one non-busy active path at least, the path selector
1681 * will be able to select it. So we consider such a pg as not busy.
1682 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001683 busy = true;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001684 list_for_each_entry(pgpath, &pg->pgpaths, list)
1685 if (pgpath->is_active) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001686 has_active = true;
Mike Snitzer9f54cec2016-02-11 21:42:28 -05001687 if (!pgpath_busy(pgpath)) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001688 busy = false;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001689 break;
1690 }
1691 }
1692
1693 if (!has_active)
1694 /*
1695 * No active path in this pg, so this pg won't be used and
1696 * the current_pg will be changed at next mapping time.
1697 * We need to try mapping to determine it.
1698 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001699 busy = false;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001700
1701out:
1702 spin_unlock_irqrestore(&m->lock, flags);
1703
1704 return busy;
1705}
1706
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707/*-----------------------------------------------------------------
1708 * Module setup
1709 *---------------------------------------------------------------*/
1710static struct target_type multipath_target = {
1711 .name = "multipath",
Mike Snitzer16f12262016-01-31 17:22:27 -05001712 .version = {1, 11, 0},
1713 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714 .module = THIS_MODULE,
1715 .ctr = multipath_ctr,
1716 .dtr = multipath_dtr,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001717 .map_rq = multipath_map,
Mike Snitzere5863d92014-12-17 21:08:12 -05001718 .clone_and_map_rq = multipath_clone_and_map,
1719 .release_clone_rq = multipath_release_clone,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001720 .rq_end_io = multipath_end_io,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721 .presuspend = multipath_presuspend,
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001722 .postsuspend = multipath_postsuspend,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001723 .resume = multipath_resume,
1724 .status = multipath_status,
1725 .message = multipath_message,
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001726 .prepare_ioctl = multipath_prepare_ioctl,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001727 .iterate_devices = multipath_iterate_devices,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001728 .busy = multipath_busy,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729};
1730
1731static int __init dm_multipath_init(void)
1732{
1733 int r;
1734
1735 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001736 _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737 if (!_mpio_cache)
1738 return -ENOMEM;
1739
1740 r = dm_register_target(&multipath_target);
1741 if (r < 0) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001742 DMERR("register failed %d", r);
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001743 r = -EINVAL;
1744 goto bad_register_target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745 }
1746
Tejun Heo4d4d66a2011-01-13 19:59:57 +00001747 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001748 if (!kmultipathd) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001749 DMERR("failed to create workqueue kmpathd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001750 r = -ENOMEM;
1751 goto bad_alloc_kmultipathd;
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001752 }
1753
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001754 /*
1755 * A separate workqueue is used to handle the device handlers
1756 * to avoid overloading existing workqueue. Overloading the
1757 * old workqueue would also create a bottleneck in the
1758 * path of the storage hardware device activation.
1759 */
Tejun Heo4d4d66a2011-01-13 19:59:57 +00001760 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
1761 WQ_MEM_RECLAIM);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001762 if (!kmpath_handlerd) {
1763 DMERR("failed to create workqueue kmpath_handlerd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001764 r = -ENOMEM;
1765 goto bad_alloc_kmpath_handlerd;
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001766 }
1767
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001768 DMINFO("version %u.%u.%u loaded",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769 multipath_target.version[0], multipath_target.version[1],
1770 multipath_target.version[2]);
1771
Johannes Thumshirnff658e92015-01-11 12:45:23 +01001772 return 0;
1773
1774bad_alloc_kmpath_handlerd:
1775 destroy_workqueue(kmultipathd);
1776bad_alloc_kmultipathd:
1777 dm_unregister_target(&multipath_target);
1778bad_register_target:
1779 kmem_cache_destroy(_mpio_cache);
1780
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781 return r;
1782}
1783
1784static void __exit dm_multipath_exit(void)
1785{
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001786 destroy_workqueue(kmpath_handlerd);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001787 destroy_workqueue(kmultipathd);
1788
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001789 dm_unregister_target(&multipath_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790 kmem_cache_destroy(_mpio_cache);
1791}
1792
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793module_init(dm_multipath_init);
1794module_exit(dm_multipath_exit);
1795
1796MODULE_DESCRIPTION(DM_NAME " multipath target");
1797MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
1798MODULE_LICENSE("GPL");