blob: c777d38f4b119dbe090b27660deda46176f6fd34 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
Mikulas Patocka586e80e2008-10-21 17:44:59 +01008#include <linux/device-mapper.h>
9
Mike Snitzer4cc96132016-05-12 16:28:10 -040010#include "dm-rq.h"
Mike Snitzer76e33fe2016-05-19 16:15:14 -040011#include "dm-bio-record.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include "dm-path-selector.h"
Mike Andersonb15546f2007-10-19 22:48:02 +010013#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070014
Mike Snitzere5863d92014-12-17 21:08:12 -050015#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/ctype.h>
17#include <linux/init.h>
18#include <linux/mempool.h>
19#include <linux/module.h>
20#include <linux/pagemap.h>
21#include <linux/slab.h>
22#include <linux/time.h>
23#include <linux/workqueue.h>
Mikulas Patocka35991652012-06-03 00:29:58 +010024#include <linux/delay.h>
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070025#include <scsi/scsi_dh.h>
Arun Sharma600634972011-07-26 16:09:06 -070026#include <linux/atomic.h>
Mike Snitzer78ce23b2016-01-31 17:38:28 -050027#include <linux/blk-mq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Alasdair G Kergon72d94862006-06-26 00:27:35 -070029#define DM_MSG_PREFIX "multipath"
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000030#define DM_PG_INIT_DELAY_MSECS 2000
31#define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
33/* Path properties */
34struct pgpath {
35 struct list_head list;
36
37 struct priority_group *pg; /* Owning PG */
38 unsigned fail_count; /* Cumulative failure count */
39
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -080040 struct dm_path path;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000041 struct delayed_work activate_path;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -050042
43 bool is_active:1; /* Path status */
Linus Torvalds1da177e2005-04-16 15:20:36 -070044};
45
46#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
47
48/*
49 * Paths are grouped into Priority Groups and numbered from 1 upwards.
50 * Each has a path selector which controls which path gets used.
51 */
52struct priority_group {
53 struct list_head list;
54
55 struct multipath *m; /* Owning multipath instance */
56 struct path_selector ps;
57
58 unsigned pg_num; /* Reference number */
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 unsigned nr_pgpaths; /* Number of paths in PG */
60 struct list_head pgpaths;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -050061
62 bool bypassed:1; /* Temporarily bypass this PG? */
Linus Torvalds1da177e2005-04-16 15:20:36 -070063};
64
65/* Multipath context */
66struct multipath {
67 struct list_head list;
68 struct dm_target *ti;
69
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070070 const char *hw_handler_name;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -070071 char *hw_handler_params;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000072
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010073 spinlock_t lock;
74
Linus Torvalds1da177e2005-04-16 15:20:36 -070075 unsigned nr_priority_groups;
76 struct list_head priority_groups;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000077
78 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
79
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 struct pgpath *current_pgpath;
81 struct priority_group *current_pg;
82 struct priority_group *next_pg; /* Switch to this PG if set */
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
Mike Snitzer518257b2016-03-17 16:32:10 -040084 unsigned long flags; /* Multipath state flags */
Mike Snitzer1fbdd2b2012-06-03 00:29:43 +010085
Dave Wysochanskic9e45582007-10-19 22:47:53 +010086 unsigned pg_init_retries; /* Number of times to retry pg_init */
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +000087 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Mike Snitzer91e968a2016-03-17 17:10:15 -040089 atomic_t nr_valid_paths; /* Total number of usable paths */
90 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */
91 atomic_t pg_init_count; /* Number of times pg_init called */
92
Mike Snitzere83068a2016-05-24 21:16:51 -040093 unsigned queue_mode;
94
Linus Torvalds1da177e2005-04-16 15:20:36 -070095 /*
Alasdair G Kergon028867a2007-07-12 17:26:32 +010096 * We must use a mempool of dm_mpath_io structs so that we
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 * can resubmit bios on error.
98 */
99 mempool_t *mpio_pool;
Mike Anderson6380f262009-12-10 23:52:21 +0000100
101 struct mutex work_mutex;
Mike Snitzer20800cb2016-03-17 17:13:10 -0400102 struct work_struct trigger_event;
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400103
104 struct work_struct process_queued_bios;
105 struct bio_list queued_bios;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106};
107
108/*
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400109 * Context information attached to each io we process.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100111struct dm_mpath_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 struct pgpath *pgpath;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100113 size_t nr_bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114};
115
116typedef int (*action_fn) (struct pgpath *pgpath);
117
Christoph Lametere18b8902006-12-06 20:33:20 -0800118static struct kmem_cache *_mpio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700120static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
David Howellsc4028952006-11-22 14:57:56 +0000121static void trigger_event(struct work_struct *work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700122static void activate_path(struct work_struct *work);
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400123static void process_queued_bios(struct work_struct *work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124
Mike Snitzer518257b2016-03-17 16:32:10 -0400125/*-----------------------------------------------
126 * Multipath state flags.
127 *-----------------------------------------------*/
128
129#define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */
130#define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */
131#define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */
132#define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */
133#define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */
134#define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */
135#define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
137/*-----------------------------------------------
138 * Allocation routines
139 *-----------------------------------------------*/
140
141static struct pgpath *alloc_pgpath(void)
142{
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700143 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
Mike Anderson224cb3e2008-08-29 09:36:09 +0200145 if (pgpath) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500146 pgpath->is_active = true;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000147 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
Mike Anderson224cb3e2008-08-29 09:36:09 +0200148 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
150 return pgpath;
151}
152
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100153static void free_pgpath(struct pgpath *pgpath)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154{
155 kfree(pgpath);
156}
157
158static struct priority_group *alloc_priority_group(void)
159{
160 struct priority_group *pg;
161
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700162 pg = kzalloc(sizeof(*pg), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700164 if (pg)
165 INIT_LIST_HEAD(&pg->pgpaths);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166
167 return pg;
168}
169
170static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
171{
172 struct pgpath *pgpath, *tmp;
173
174 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
175 list_del(&pgpath->list);
176 dm_put_device(ti, pgpath->path.dev);
177 free_pgpath(pgpath);
178 }
179}
180
181static void free_priority_group(struct priority_group *pg,
182 struct dm_target *ti)
183{
184 struct path_selector *ps = &pg->ps;
185
186 if (ps->type) {
187 ps->type->destroy(ps);
188 dm_put_path_selector(ps->type);
189 }
190
191 free_pgpaths(&pg->pgpaths, ti);
192 kfree(pg);
193}
194
Mike Snitzere83068a2016-05-24 21:16:51 -0400195static struct multipath *alloc_multipath(struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196{
197 struct multipath *m;
198
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700199 m = kzalloc(sizeof(*m), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 if (m) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 INIT_LIST_HEAD(&m->priority_groups);
202 spin_lock_init(&m->lock);
Mike Snitzer518257b2016-03-17 16:32:10 -0400203 set_bit(MPATHF_QUEUE_IO, &m->flags);
Mike Snitzer91e968a2016-03-17 17:10:15 -0400204 atomic_set(&m->nr_valid_paths, 0);
205 atomic_set(&m->pg_init_in_progress, 0);
206 atomic_set(&m->pg_init_count, 0);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000207 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
David Howellsc4028952006-11-22 14:57:56 +0000208 INIT_WORK(&m->trigger_event, trigger_event);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000209 init_waitqueue_head(&m->pg_init_wait);
Mike Anderson6380f262009-12-10 23:52:21 +0000210 mutex_init(&m->work_mutex);
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500211
212 m->mpio_pool = NULL;
Mike Snitzere83068a2016-05-24 21:16:51 -0400213 m->queue_mode = DM_TYPE_NONE;
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400214
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700215 m->ti = ti;
216 ti->private = m;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 }
218
219 return m;
220}
221
Mike Snitzere83068a2016-05-24 21:16:51 -0400222static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
223{
224 if (m->queue_mode == DM_TYPE_NONE) {
225 /*
226 * Default to request-based.
227 */
228 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
229 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
230 else
231 m->queue_mode = DM_TYPE_REQUEST_BASED;
232 }
233
234 if (m->queue_mode == DM_TYPE_REQUEST_BASED) {
235 unsigned min_ios = dm_get_reserved_rq_based_ios();
236
237 m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache);
238 if (!m->mpio_pool)
239 return -ENOMEM;
240 }
241 else if (m->queue_mode == DM_TYPE_BIO_BASED) {
242 INIT_WORK(&m->process_queued_bios, process_queued_bios);
243 /*
244 * bio-based doesn't support any direct scsi_dh management;
245 * it just discovers if a scsi_dh is attached.
246 */
247 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
248 }
249
250 dm_table_set_type(ti->table, m->queue_mode);
251
252 return 0;
253}
254
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255static void free_multipath(struct multipath *m)
256{
257 struct priority_group *pg, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258
259 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
260 list_del(&pg->list);
261 free_priority_group(pg, m->ti);
262 }
263
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700264 kfree(m->hw_handler_name);
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700265 kfree(m->hw_handler_params);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 mempool_destroy(m->mpio_pool);
267 kfree(m);
268}
269
Mike Snitzer2eff1922016-02-03 09:13:14 -0500270static struct dm_mpath_io *get_mpio(union map_info *info)
271{
272 return info->ptr;
273}
274
275static struct dm_mpath_io *set_mpio(struct multipath *m, union map_info *info)
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100276{
277 struct dm_mpath_io *mpio;
278
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500279 if (!m->mpio_pool) {
280 /* Use blk-mq pdu memory requested via per_io_data_size */
Mike Snitzer2eff1922016-02-03 09:13:14 -0500281 mpio = get_mpio(info);
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500282 memset(mpio, 0, sizeof(*mpio));
283 return mpio;
284 }
285
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100286 mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
287 if (!mpio)
Mike Snitzer2eff1922016-02-03 09:13:14 -0500288 return NULL;
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100289
290 memset(mpio, 0, sizeof(*mpio));
291 info->ptr = mpio;
292
Mike Snitzer2eff1922016-02-03 09:13:14 -0500293 return mpio;
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100294}
295
Mike Snitzer2eff1922016-02-03 09:13:14 -0500296static void clear_request_fn_mpio(struct multipath *m, union map_info *info)
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100297{
Mike Snitzer2eff1922016-02-03 09:13:14 -0500298 /* Only needed for non blk-mq (.request_fn) multipath */
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500299 if (m->mpio_pool) {
300 struct dm_mpath_io *mpio = info->ptr;
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100301
Mike Snitzer8637a6b2016-01-31 12:08:36 -0500302 info->ptr = NULL;
303 mempool_free(mpio, m->mpio_pool);
304 }
Jun'ichi Nomura466891f2012-03-28 18:41:25 +0100305}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
Mike Snitzerbf661be2016-05-24 15:48:08 -0400307static size_t multipath_per_bio_data_size(void)
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400308{
Mike Snitzerbf661be2016-05-24 15:48:08 -0400309 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400310}
311
Mike Snitzerbf661be2016-05-24 15:48:08 -0400312static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
313{
314 return dm_per_bio_data(bio, multipath_per_bio_data_size());
315}
316
317static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio)
318{
319 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
320 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
321 void *bio_details = mpio + 1;
322
323 return bio_details;
324}
325
326static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p,
327 struct dm_bio_details **bio_details_p)
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400328{
329 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
Mike Snitzerbf661be2016-05-24 15:48:08 -0400330 struct dm_bio_details *bio_details = get_bio_details_from_bio(bio);
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400331
332 memset(mpio, 0, sizeof(*mpio));
Mike Snitzerbf661be2016-05-24 15:48:08 -0400333 memset(bio_details, 0, sizeof(*bio_details));
334 dm_bio_record(bio_details, bio);
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400335
Mike Snitzerbf661be2016-05-24 15:48:08 -0400336 if (mpio_p)
337 *mpio_p = mpio;
338 if (bio_details_p)
339 *bio_details_p = bio_details;
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400340}
341
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342/*-----------------------------------------------
343 * Path selection
344 *-----------------------------------------------*/
345
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100346static int __pg_init_all_paths(struct multipath *m)
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000347{
348 struct pgpath *pgpath;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000349 unsigned long pg_init_delay = 0;
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000350
Mike Snitzer91e968a2016-03-17 17:10:15 -0400351 if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100352 return 0;
Hannes Reinecke17f4ff42014-02-28 15:33:42 +0100353
Mike Snitzer91e968a2016-03-17 17:10:15 -0400354 atomic_inc(&m->pg_init_count);
Mike Snitzer518257b2016-03-17 16:32:10 -0400355 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +0100356
357 /* Check here to reset pg_init_required */
358 if (!m->current_pg)
359 return 0;
360
Mike Snitzer518257b2016-03-17 16:32:10 -0400361 if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000362 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
363 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000364 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
365 /* Skip failed paths */
366 if (!pgpath->is_active)
367 continue;
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +0000368 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
369 pg_init_delay))
Mike Snitzer91e968a2016-03-17 17:10:15 -0400370 atomic_inc(&m->pg_init_in_progress);
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000371 }
Mike Snitzer91e968a2016-03-17 17:10:15 -0400372 return atomic_read(&m->pg_init_in_progress);
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000373}
374
Mike Snitzer2da16102016-03-17 18:38:17 -0400375static int pg_init_all_paths(struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
Mike Snitzer2da16102016-03-17 18:38:17 -0400377 int r;
378 unsigned long flags;
379
380 spin_lock_irqsave(&m->lock, flags);
381 r = __pg_init_all_paths(m);
382 spin_unlock_irqrestore(&m->lock, flags);
383
384 return r;
385}
386
387static void __switch_pg(struct multipath *m, struct priority_group *pg)
388{
389 m->current_pg = pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390
391 /* Must we initialise the PG first, and queue I/O till it's ready? */
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700392 if (m->hw_handler_name) {
Mike Snitzer518257b2016-03-17 16:32:10 -0400393 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
394 set_bit(MPATHF_QUEUE_IO, &m->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 } else {
Mike Snitzer518257b2016-03-17 16:32:10 -0400396 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
397 clear_bit(MPATHF_QUEUE_IO, &m->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 }
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100399
Mike Snitzer91e968a2016-03-17 17:10:15 -0400400 atomic_set(&m->pg_init_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401}
402
Mike Snitzer2da16102016-03-17 18:38:17 -0400403static struct pgpath *choose_path_in_pg(struct multipath *m,
404 struct priority_group *pg,
405 size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406{
Mike Snitzer2da16102016-03-17 18:38:17 -0400407 unsigned long flags;
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -0800408 struct dm_path *path;
Mike Snitzer2da16102016-03-17 18:38:17 -0400409 struct pgpath *pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410
Mike Snitzer90a43232016-02-17 21:29:17 -0500411 path = pg->ps.type->select_path(&pg->ps, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412 if (!path)
Mike Snitzer2da16102016-03-17 18:38:17 -0400413 return ERR_PTR(-ENXIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414
Mike Snitzer2da16102016-03-17 18:38:17 -0400415 pgpath = path_to_pgpath(path);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416
Mike Snitzer2da16102016-03-17 18:38:17 -0400417 if (unlikely(lockless_dereference(m->current_pg) != pg)) {
418 /* Only update current_pgpath if pg changed */
419 spin_lock_irqsave(&m->lock, flags);
420 m->current_pgpath = pgpath;
421 __switch_pg(m, pg);
422 spin_unlock_irqrestore(&m->lock, flags);
423 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424
Mike Snitzer2da16102016-03-17 18:38:17 -0400425 return pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426}
427
Mike Snitzer2da16102016-03-17 18:38:17 -0400428static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429{
Mike Snitzer2da16102016-03-17 18:38:17 -0400430 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431 struct priority_group *pg;
Mike Snitzer2da16102016-03-17 18:38:17 -0400432 struct pgpath *pgpath;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500433 bool bypassed = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434
Mike Snitzer91e968a2016-03-17 17:10:15 -0400435 if (!atomic_read(&m->nr_valid_paths)) {
Mike Snitzer518257b2016-03-17 16:32:10 -0400436 clear_bit(MPATHF_QUEUE_IO, &m->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 goto failed;
Benjamin Marzinski1f271972014-08-13 13:53:42 -0500438 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
440 /* Were we instructed to switch PG? */
Mike Snitzer2da16102016-03-17 18:38:17 -0400441 if (lockless_dereference(m->next_pg)) {
442 spin_lock_irqsave(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 pg = m->next_pg;
Mike Snitzer2da16102016-03-17 18:38:17 -0400444 if (!pg) {
445 spin_unlock_irqrestore(&m->lock, flags);
446 goto check_current_pg;
447 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 m->next_pg = NULL;
Mike Snitzer2da16102016-03-17 18:38:17 -0400449 spin_unlock_irqrestore(&m->lock, flags);
450 pgpath = choose_path_in_pg(m, pg, nr_bytes);
451 if (!IS_ERR_OR_NULL(pgpath))
452 return pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 }
454
455 /* Don't change PG until it has no remaining paths */
Mike Snitzer2da16102016-03-17 18:38:17 -0400456check_current_pg:
457 pg = lockless_dereference(m->current_pg);
458 if (pg) {
459 pgpath = choose_path_in_pg(m, pg, nr_bytes);
460 if (!IS_ERR_OR_NULL(pgpath))
461 return pgpath;
462 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463
464 /*
465 * Loop through priority groups until we find a valid path.
466 * First time we skip PGs marked 'bypassed'.
Mike Christief220fd42012-06-03 00:29:45 +0100467 * Second time we only try the ones we skipped, but set
468 * pg_init_delay_retry so we do not hammer controllers.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 */
470 do {
471 list_for_each_entry(pg, &m->priority_groups, list) {
472 if (pg->bypassed == bypassed)
473 continue;
Mike Snitzer2da16102016-03-17 18:38:17 -0400474 pgpath = choose_path_in_pg(m, pg, nr_bytes);
475 if (!IS_ERR_OR_NULL(pgpath)) {
Mike Christief220fd42012-06-03 00:29:45 +0100476 if (!bypassed)
Mike Snitzer518257b2016-03-17 16:32:10 -0400477 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
Mike Snitzer2da16102016-03-17 18:38:17 -0400478 return pgpath;
Mike Christief220fd42012-06-03 00:29:45 +0100479 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 }
481 } while (bypassed--);
482
483failed:
Mike Snitzer2da16102016-03-17 18:38:17 -0400484 spin_lock_irqsave(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 m->current_pgpath = NULL;
486 m->current_pg = NULL;
Mike Snitzer2da16102016-03-17 18:38:17 -0400487 spin_unlock_irqrestore(&m->lock, flags);
488
489 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490}
491
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800492/*
493 * Check whether bios must be queued in the device-mapper core rather
494 * than here in the target.
495 *
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800496 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
497 * same value then we are not between multipath_presuspend()
498 * and multipath_resume() calls and we have no need to check
499 * for the DMF_NOFLUSH_SUSPENDING flag.
500 */
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400501static bool __must_push_back(struct multipath *m)
502{
503 return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) !=
504 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) &&
505 dm_noflush_suspending(m->ti));
506}
507
508static bool must_push_back_rq(struct multipath *m)
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800509{
Mike Snitzer1814f2e2016-07-25 21:08:51 -0400510 bool r;
511 unsigned long flags;
512
513 spin_lock_irqsave(&m->lock, flags);
514 r = (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) ||
515 __must_push_back(m));
516 spin_unlock_irqrestore(&m->lock, flags);
517
518 return r;
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400519}
520
521static bool must_push_back_bio(struct multipath *m)
522{
Mike Snitzer1814f2e2016-07-25 21:08:51 -0400523 bool r;
524 unsigned long flags;
525
526 spin_lock_irqsave(&m->lock, flags);
527 r = __must_push_back(m);
528 spin_unlock_irqrestore(&m->lock, flags);
529
530 return r;
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800531}
532
Hannes Reinecke36fcffc2014-02-28 15:33:47 +0100533/*
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400534 * Map cloned requests (request-based multipath)
Hannes Reinecke36fcffc2014-02-28 15:33:47 +0100535 */
Mike Snitzere5863d92014-12-17 21:08:12 -0500536static int __multipath_map(struct dm_target *ti, struct request *clone,
537 union map_info *map_context,
538 struct request *rq, struct request **__clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539{
Mike Snitzer7943bd62016-02-02 21:53:15 -0500540 struct multipath *m = ti->private;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100541 int r = DM_MAPIO_REQUEUE;
Mike Snitzere5863d92014-12-17 21:08:12 -0500542 size_t nr_bytes = clone ? blk_rq_bytes(clone) : blk_rq_bytes(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 struct pgpath *pgpath;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100544 struct block_device *bdev;
Hannes Reineckee3bde042014-02-28 15:33:46 +0100545 struct dm_mpath_io *mpio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 /* Do we need to select a new pgpath? */
Mike Snitzer2da16102016-03-17 18:38:17 -0400548 pgpath = lockless_dereference(m->current_pgpath);
549 if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
550 pgpath = choose_pgpath(m, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100552 if (!pgpath) {
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400553 if (!must_push_back_rq(m))
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100554 r = -EIO; /* Failed */
Mike Snitzer2da16102016-03-17 18:38:17 -0400555 return r;
Mike Snitzer518257b2016-03-17 16:32:10 -0400556 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
557 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
Mike Snitzer2da16102016-03-17 18:38:17 -0400558 pg_init_all_paths(m);
559 return r;
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100560 }
Mike Snitzer6afbc012014-07-08 11:55:09 -0400561
Mike Snitzer2eff1922016-02-03 09:13:14 -0500562 mpio = set_mpio(m, map_context);
563 if (!mpio)
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100564 /* ENOMEM, requeue */
Mike Snitzer2da16102016-03-17 18:38:17 -0400565 return r;
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100566
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100567 mpio->pgpath = pgpath;
568 mpio->nr_bytes = nr_bytes;
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600569
570 bdev = pgpath->path.dev->bdev;
571
Mike Snitzere5863d92014-12-17 21:08:12 -0500572 if (clone) {
Mike Snitzerc5248f72016-02-20 14:02:49 -0500573 /*
574 * Old request-based interface: allocated clone is passed in.
575 * Used by: .request_fn stacked on .request_fn path(s).
576 */
Mike Snitzere5863d92014-12-17 21:08:12 -0500577 clone->q = bdev_get_queue(bdev);
578 clone->rq_disk = bdev->bd_disk;
579 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
580 } else {
Mike Snitzereca7ee62016-02-20 13:45:38 -0500581 /*
582 * blk-mq request-based interface; used by both:
583 * .request_fn stacked on blk-mq path(s) and
584 * blk-mq stacked on blk-mq path(s).
585 */
Mike Snitzer78ce23b2016-01-31 17:38:28 -0500586 *__clone = blk_mq_alloc_request(bdev_get_queue(bdev),
587 rq_data_dir(rq), BLK_MQ_REQ_NOWAIT);
Mike Snitzer4c6dd532015-05-27 15:23:56 -0400588 if (IS_ERR(*__clone)) {
Mike Snitzere5863d92014-12-17 21:08:12 -0500589 /* ENOMEM, requeue */
Mike Snitzer2eff1922016-02-03 09:13:14 -0500590 clear_request_fn_mpio(m, map_context);
Mike Snitzere5863d92014-12-17 21:08:12 -0500591 return r;
Mike Snitzer4c6dd532015-05-27 15:23:56 -0400592 }
Mike Snitzere5863d92014-12-17 21:08:12 -0500593 (*__clone)->bio = (*__clone)->biotail = NULL;
594 (*__clone)->rq_disk = bdev->bd_disk;
595 (*__clone)->cmd_flags |= REQ_FAILFAST_TRANSPORT;
596 }
597
Mike Snitzer9bf59a62014-02-28 15:33:48 +0100598 if (pgpath->pg->ps.type->start_io)
599 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
600 &pgpath->path,
601 nr_bytes);
Keith Busch2eb6e1e2014-10-17 17:46:36 -0600602 return DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603}
604
Mike Snitzere5863d92014-12-17 21:08:12 -0500605static int multipath_map(struct dm_target *ti, struct request *clone,
606 union map_info *map_context)
607{
608 return __multipath_map(ti, clone, map_context, NULL, NULL);
609}
610
611static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
612 union map_info *map_context,
613 struct request **clone)
614{
615 return __multipath_map(ti, NULL, map_context, rq, clone);
616}
617
618static void multipath_release_clone(struct request *clone)
619{
Mike Snitzer78ce23b2016-01-31 17:38:28 -0500620 blk_mq_free_request(clone);
Mike Snitzere5863d92014-12-17 21:08:12 -0500621}
622
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623/*
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400624 * Map cloned bios (bio-based multipath)
625 */
626static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio)
627{
628 size_t nr_bytes = bio->bi_iter.bi_size;
629 struct pgpath *pgpath;
630 unsigned long flags;
631 bool queue_io;
632
633 /* Do we need to select a new pgpath? */
634 pgpath = lockless_dereference(m->current_pgpath);
635 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
636 if (!pgpath || !queue_io)
637 pgpath = choose_pgpath(m, nr_bytes);
638
639 if ((pgpath && queue_io) ||
640 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
641 /* Queue for the daemon to resubmit */
642 spin_lock_irqsave(&m->lock, flags);
643 bio_list_add(&m->queued_bios, bio);
644 spin_unlock_irqrestore(&m->lock, flags);
645 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
646 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
647 pg_init_all_paths(m);
648 else if (!queue_io)
649 queue_work(kmultipathd, &m->process_queued_bios);
650 return DM_MAPIO_SUBMITTED;
651 }
652
653 if (!pgpath) {
654 if (!must_push_back_bio(m))
655 return -EIO;
656 return DM_MAPIO_REQUEUE;
657 }
658
659 mpio->pgpath = pgpath;
660 mpio->nr_bytes = nr_bytes;
661
662 bio->bi_error = 0;
663 bio->bi_bdev = pgpath->path.dev->bdev;
Jens Axboe1eff9d32016-08-05 15:35:16 -0600664 bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400665
666 if (pgpath->pg->ps.type->start_io)
667 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
668 &pgpath->path,
669 nr_bytes);
670 return DM_MAPIO_REMAPPED;
671}
672
673static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
674{
675 struct multipath *m = ti->private;
Mike Snitzerbf661be2016-05-24 15:48:08 -0400676 struct dm_mpath_io *mpio = NULL;
677
678 multipath_init_per_bio_data(bio, &mpio, NULL);
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400679
680 return __multipath_map_bio(m, bio, mpio);
681}
682
683static void process_queued_bios_list(struct multipath *m)
684{
Mike Snitzere83068a2016-05-24 21:16:51 -0400685 if (m->queue_mode == DM_TYPE_BIO_BASED)
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400686 queue_work(kmultipathd, &m->process_queued_bios);
687}
688
689static void process_queued_bios(struct work_struct *work)
690{
691 int r;
692 unsigned long flags;
693 struct bio *bio;
694 struct bio_list bios;
695 struct blk_plug plug;
696 struct multipath *m =
697 container_of(work, struct multipath, process_queued_bios);
698
699 bio_list_init(&bios);
700
701 spin_lock_irqsave(&m->lock, flags);
702
703 if (bio_list_empty(&m->queued_bios)) {
704 spin_unlock_irqrestore(&m->lock, flags);
705 return;
706 }
707
708 bio_list_merge(&bios, &m->queued_bios);
709 bio_list_init(&m->queued_bios);
710
711 spin_unlock_irqrestore(&m->lock, flags);
712
713 blk_start_plug(&plug);
714 while ((bio = bio_list_pop(&bios))) {
715 r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio));
716 if (r < 0 || r == DM_MAPIO_REQUEUE) {
717 bio->bi_error = r;
718 bio_endio(bio);
719 } else if (r == DM_MAPIO_REMAPPED)
720 generic_make_request(bio);
721 }
722 blk_finish_plug(&plug);
723}
724
725/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 * If we run out of usable paths, should we queue I/O or error it?
727 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -0500728static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
729 bool save_old_value)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730{
731 unsigned long flags;
732
733 spin_lock_irqsave(&m->lock, flags);
734
Mike Snitzer518257b2016-03-17 16:32:10 -0400735 if (save_old_value) {
736 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
737 set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
738 else
739 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
740 } else {
741 if (queue_if_no_path)
742 set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
743 else
744 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
745 }
746 if (queue_if_no_path)
747 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700748 else
Mike Snitzer518257b2016-03-17 16:32:10 -0400749 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
750
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 spin_unlock_irqrestore(&m->lock, flags);
752
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400753 if (!queue_if_no_path) {
Hannes Reinecke63d832c2014-05-26 14:45:39 +0200754 dm_table_run_md_queue_async(m->ti->table);
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400755 process_queued_bios_list(m);
756 }
Hannes Reinecke63d832c2014-05-26 14:45:39 +0200757
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 return 0;
759}
760
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761/*
762 * An event is triggered whenever a path is taken out of use.
763 * Includes path failure and PG bypass.
764 */
David Howellsc4028952006-11-22 14:57:56 +0000765static void trigger_event(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766{
David Howellsc4028952006-11-22 14:57:56 +0000767 struct multipath *m =
768 container_of(work, struct multipath, trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769
770 dm_table_event(m->ti->table);
771}
772
773/*-----------------------------------------------------------------
774 * Constructor/argument parsing:
775 * <#multipath feature args> [<arg>]*
776 * <#hw_handler args> [hw_handler [<arg>]*]
777 * <#priority groups>
778 * <initial priority group>
779 * [<selector> <#selector args> [<arg>]*
780 * <#paths> <#per-path selector args>
781 * [<path> [<arg>]* ]+ ]+
782 *---------------------------------------------------------------*/
Mike Snitzer498f0102011-08-02 12:32:04 +0100783static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 struct dm_target *ti)
785{
786 int r;
787 struct path_selector_type *pst;
788 unsigned ps_argc;
789
Mike Snitzer498f0102011-08-02 12:32:04 +0100790 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700791 {0, 1024, "invalid number of path selector args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 };
793
Mike Snitzer498f0102011-08-02 12:32:04 +0100794 pst = dm_get_path_selector(dm_shift_arg(as));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 if (!pst) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700796 ti->error = "unknown path selector type";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 return -EINVAL;
798 }
799
Mike Snitzer498f0102011-08-02 12:32:04 +0100800 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100801 if (r) {
802 dm_put_path_selector(pst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 return -EINVAL;
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100804 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805
806 r = pst->create(&pg->ps, ps_argc, as->argv);
807 if (r) {
808 dm_put_path_selector(pst);
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700809 ti->error = "path selector constructor failed";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 return r;
811 }
812
813 pg->ps.type = pst;
Mike Snitzer498f0102011-08-02 12:32:04 +0100814 dm_consume_args(as, ps_argc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
816 return 0;
817}
818
Mike Snitzer498f0102011-08-02 12:32:04 +0100819static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 struct dm_target *ti)
821{
822 int r;
823 struct pgpath *p;
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700824 struct multipath *m = ti->private;
Mike Snitzera58a9352012-07-27 15:08:04 +0100825 struct request_queue *q = NULL;
826 const char *attached_handler_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827
828 /* we need at least a path arg */
829 if (as->argc < 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700830 ti->error = "no device given";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100831 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832 }
833
834 p = alloc_pgpath();
835 if (!p)
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100836 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837
Mike Snitzer498f0102011-08-02 12:32:04 +0100838 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +0000839 &p->path.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700841 ti->error = "error getting device";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842 goto bad;
843 }
844
Mike Snitzer518257b2016-03-17 16:32:10 -0400845 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name)
Mike Snitzera58a9352012-07-27 15:08:04 +0100846 q = bdev_get_queue(p->path.dev->bdev);
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100847
Mike Snitzer518257b2016-03-17 16:32:10 -0400848 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200849retain:
Mike Snitzera58a9352012-07-27 15:08:04 +0100850 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
851 if (attached_handler_name) {
852 /*
853 * Reset hw_handler_name to match the attached handler
854 * and clear any hw_handler_params associated with the
855 * ignored handler.
856 *
857 * NB. This modifies the table line to show the actual
858 * handler instead of the original table passed in.
859 */
860 kfree(m->hw_handler_name);
861 m->hw_handler_name = attached_handler_name;
862
863 kfree(m->hw_handler_params);
864 m->hw_handler_params = NULL;
865 }
866 }
867
868 if (m->hw_handler_name) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100869 r = scsi_dh_attach(q, m->hw_handler_name);
870 if (r == -EBUSY) {
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200871 char b[BDEVNAME_SIZE];
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100872
Christoph Hellwig1bab0de2015-08-27 14:16:54 +0200873 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
874 bdevname(p->path.dev->bdev, b));
875 goto retain;
876 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700877 if (r < 0) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100878 ti->error = "error attaching hardware handler";
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700879 dm_put_device(ti, p->path.dev);
880 goto bad;
881 }
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700882
883 if (m->hw_handler_params) {
884 r = scsi_dh_set_params(q, m->hw_handler_params);
885 if (r < 0) {
886 ti->error = "unable to set hardware "
887 "handler parameters";
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700888 dm_put_device(ti, p->path.dev);
889 goto bad;
890 }
891 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700892 }
893
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
895 if (r) {
896 dm_put_device(ti, p->path.dev);
897 goto bad;
898 }
899
900 return p;
901
902 bad:
903 free_pgpath(p);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100904 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905}
906
Mike Snitzer498f0102011-08-02 12:32:04 +0100907static struct priority_group *parse_priority_group(struct dm_arg_set *as,
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700908 struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909{
Mike Snitzer498f0102011-08-02 12:32:04 +0100910 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700911 {1, 1024, "invalid number of paths"},
912 {0, 1024, "invalid number of selector args"}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 };
914
915 int r;
Mike Snitzer498f0102011-08-02 12:32:04 +0100916 unsigned i, nr_selector_args, nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 struct priority_group *pg;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700918 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919
920 if (as->argc < 2) {
921 as->argc = 0;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100922 ti->error = "not enough priority group arguments";
923 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 }
925
926 pg = alloc_priority_group();
927 if (!pg) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700928 ti->error = "couldn't allocate priority group";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100929 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930 }
931 pg->m = m;
932
933 r = parse_path_selector(as, pg, ti);
934 if (r)
935 goto bad;
936
937 /*
938 * read the paths
939 */
Mike Snitzer498f0102011-08-02 12:32:04 +0100940 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 if (r)
942 goto bad;
943
Mike Snitzer498f0102011-08-02 12:32:04 +0100944 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945 if (r)
946 goto bad;
947
Mike Snitzer498f0102011-08-02 12:32:04 +0100948 nr_args = 1 + nr_selector_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949 for (i = 0; i < pg->nr_pgpaths; i++) {
950 struct pgpath *pgpath;
Mike Snitzer498f0102011-08-02 12:32:04 +0100951 struct dm_arg_set path_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952
Mike Snitzer498f0102011-08-02 12:32:04 +0100953 if (as->argc < nr_args) {
Mikulas Patocka148acff2008-07-21 12:00:30 +0100954 ti->error = "not enough path parameters";
Alasdair G Kergon6bbf79a2010-08-12 04:13:49 +0100955 r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956 goto bad;
Mikulas Patocka148acff2008-07-21 12:00:30 +0100957 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958
Mike Snitzer498f0102011-08-02 12:32:04 +0100959 path_args.argc = nr_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 path_args.argv = as->argv;
961
962 pgpath = parse_path(&path_args, &pg->ps, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100963 if (IS_ERR(pgpath)) {
964 r = PTR_ERR(pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 goto bad;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100966 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967
968 pgpath->pg = pg;
969 list_add_tail(&pgpath->list, &pg->pgpaths);
Mike Snitzer498f0102011-08-02 12:32:04 +0100970 dm_consume_args(as, nr_args);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971 }
972
973 return pg;
974
975 bad:
976 free_priority_group(pg, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100977 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978}
979
Mike Snitzer498f0102011-08-02 12:32:04 +0100980static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 unsigned hw_argc;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700983 int ret;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700984 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985
Mike Snitzer498f0102011-08-02 12:32:04 +0100986 static struct dm_arg _args[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700987 {0, 1024, "invalid number of hardware handler args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 };
989
Mike Snitzer498f0102011-08-02 12:32:04 +0100990 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 return -EINVAL;
992
993 if (!hw_argc)
994 return 0;
995
Mike Snitzere83068a2016-05-24 21:16:51 -0400996 if (m->queue_mode == DM_TYPE_BIO_BASED) {
Mike Snitzer76e33fe2016-05-19 16:15:14 -0400997 dm_consume_args(as, hw_argc);
998 DMERR("bio-based multipath doesn't allow hardware handler args");
999 return 0;
1000 }
1001
Mike Snitzer498f0102011-08-02 12:32:04 +01001002 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
Chandra Seetharaman14e98c52008-11-13 23:39:06 +00001003
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -07001004 if (hw_argc > 1) {
1005 char *p;
1006 int i, j, len = 4;
1007
1008 for (i = 0; i <= hw_argc - 2; i++)
1009 len += strlen(as->argv[i]) + 1;
1010 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
1011 if (!p) {
1012 ti->error = "memory allocation failed";
1013 ret = -ENOMEM;
1014 goto fail;
1015 }
1016 j = sprintf(p, "%d", hw_argc - 1);
1017 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
1018 j = sprintf(p, "%s", as->argv[i]);
1019 }
Mike Snitzer498f0102011-08-02 12:32:04 +01001020 dm_consume_args(as, hw_argc - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021
1022 return 0;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -07001023fail:
1024 kfree(m->hw_handler_name);
1025 m->hw_handler_name = NULL;
1026 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027}
1028
Mike Snitzer498f0102011-08-02 12:32:04 +01001029static int parse_features(struct dm_arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030{
1031 int r;
1032 unsigned argc;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -07001033 struct dm_target *ti = m->ti;
Mike Snitzer498f0102011-08-02 12:32:04 +01001034 const char *arg_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035
Mike Snitzer498f0102011-08-02 12:32:04 +01001036 static struct dm_arg _args[] = {
Mike Snitzere83068a2016-05-24 21:16:51 -04001037 {0, 8, "invalid number of feature args"},
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001038 {1, 50, "pg_init_retries must be between 1 and 50"},
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001039 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040 };
1041
Mike Snitzer498f0102011-08-02 12:32:04 +01001042 r = dm_read_arg_group(_args, as, &argc, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 if (r)
1044 return -EINVAL;
1045
1046 if (!argc)
1047 return 0;
1048
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001049 do {
Mike Snitzer498f0102011-08-02 12:32:04 +01001050 arg_name = dm_shift_arg(as);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001051 argc--;
1052
Mike Snitzer498f0102011-08-02 12:32:04 +01001053 if (!strcasecmp(arg_name, "queue_if_no_path")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001054 r = queue_if_no_path(m, true, false);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001055 continue;
1056 }
1057
Mike Snitzera58a9352012-07-27 15:08:04 +01001058 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
Mike Snitzer518257b2016-03-17 16:32:10 -04001059 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
Mike Snitzera58a9352012-07-27 15:08:04 +01001060 continue;
1061 }
1062
Mike Snitzer498f0102011-08-02 12:32:04 +01001063 if (!strcasecmp(arg_name, "pg_init_retries") &&
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001064 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +01001065 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001066 argc--;
1067 continue;
1068 }
1069
Mike Snitzer498f0102011-08-02 12:32:04 +01001070 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001071 (argc >= 1)) {
Mike Snitzer498f0102011-08-02 12:32:04 +01001072 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001073 argc--;
1074 continue;
1075 }
1076
Mike Snitzere83068a2016-05-24 21:16:51 -04001077 if (!strcasecmp(arg_name, "queue_mode") &&
1078 (argc >= 1)) {
1079 const char *queue_mode_name = dm_shift_arg(as);
1080
1081 if (!strcasecmp(queue_mode_name, "bio"))
1082 m->queue_mode = DM_TYPE_BIO_BASED;
1083 else if (!strcasecmp(queue_mode_name, "rq"))
1084 m->queue_mode = DM_TYPE_REQUEST_BASED;
1085 else if (!strcasecmp(queue_mode_name, "mq"))
1086 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1087 else {
1088 ti->error = "Unknown 'queue_mode' requested";
1089 r = -EINVAL;
1090 }
1091 argc--;
1092 continue;
1093 }
1094
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095 ti->error = "Unrecognised multipath feature request";
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001096 r = -EINVAL;
1097 } while (argc && !r);
1098
1099 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100}
1101
Mike Snitzere83068a2016-05-24 21:16:51 -04001102static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103{
Mike Snitzer498f0102011-08-02 12:32:04 +01001104 /* target arguments */
1105 static struct dm_arg _args[] = {
Mike Snitzera490a072011-03-24 13:54:33 +00001106 {0, 1024, "invalid number of priority groups"},
1107 {0, 1024, "invalid initial priority group number"},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 };
1109
1110 int r;
1111 struct multipath *m;
Mike Snitzer498f0102011-08-02 12:32:04 +01001112 struct dm_arg_set as;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 unsigned pg_count = 0;
1114 unsigned next_pg_num;
1115
1116 as.argc = argc;
1117 as.argv = argv;
1118
Mike Snitzere83068a2016-05-24 21:16:51 -04001119 m = alloc_multipath(ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 if (!m) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001121 ti->error = "can't allocate multipath";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 return -EINVAL;
1123 }
1124
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -07001125 r = parse_features(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 if (r)
1127 goto bad;
1128
Mike Snitzere83068a2016-05-24 21:16:51 -04001129 r = alloc_multipath_stage2(ti, m);
1130 if (r)
1131 goto bad;
1132
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -07001133 r = parse_hw_handler(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 if (r)
1135 goto bad;
1136
Mike Snitzer498f0102011-08-02 12:32:04 +01001137 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 if (r)
1139 goto bad;
1140
Mike Snitzer498f0102011-08-02 12:32:04 +01001141 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142 if (r)
1143 goto bad;
1144
Mike Snitzera490a072011-03-24 13:54:33 +00001145 if ((!m->nr_priority_groups && next_pg_num) ||
1146 (m->nr_priority_groups && !next_pg_num)) {
1147 ti->error = "invalid initial priority group";
1148 r = -EINVAL;
1149 goto bad;
1150 }
1151
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 /* parse the priority groups */
1153 while (as.argc) {
1154 struct priority_group *pg;
Mike Snitzer91e968a2016-03-17 17:10:15 -04001155 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -07001157 pg = parse_priority_group(&as, m);
Benjamin Marzinski01460f32008-10-10 13:36:57 +01001158 if (IS_ERR(pg)) {
1159 r = PTR_ERR(pg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 goto bad;
1161 }
1162
Mike Snitzer91e968a2016-03-17 17:10:15 -04001163 nr_valid_paths += pg->nr_pgpaths;
1164 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1165
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 list_add_tail(&pg->list, &m->priority_groups);
1167 pg_count++;
1168 pg->pg_num = pg_count;
1169 if (!--next_pg_num)
1170 m->next_pg = pg;
1171 }
1172
1173 if (pg_count != m->nr_priority_groups) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001174 ti->error = "priority group count mismatch";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 r = -EINVAL;
1176 goto bad;
1177 }
1178
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001179 ti->num_flush_bios = 1;
1180 ti->num_discard_bios = 1;
Mike Snitzer042bcef2013-05-10 14:37:16 +01001181 ti->num_write_same_bios = 1;
Mike Snitzere83068a2016-05-24 21:16:51 -04001182 if (m->queue_mode == DM_TYPE_BIO_BASED)
Mike Snitzerbf661be2016-05-24 15:48:08 -04001183 ti->per_io_data_size = multipath_per_bio_data_size();
Mike Snitzere83068a2016-05-24 21:16:51 -04001184 else if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
Mike Snitzer8637a6b2016-01-31 12:08:36 -05001185 ti->per_io_data_size = sizeof(struct dm_mpath_io);
Mikulas Patocka86279212009-06-22 10:12:24 +01001186
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187 return 0;
1188
1189 bad:
1190 free_multipath(m);
1191 return r;
1192}
1193
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001194static void multipath_wait_for_pg_init_completion(struct multipath *m)
1195{
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07001196 DEFINE_WAIT(wait);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001197
1198 while (1) {
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07001199 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001200
Mike Snitzer91e968a2016-03-17 17:10:15 -04001201 if (!atomic_read(&m->pg_init_in_progress))
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001202 break;
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001203
1204 io_schedule();
1205 }
Bart Van Assche9f4c3f82016-08-31 15:16:43 -07001206 finish_wait(&m->pg_init_wait, &wait);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001207}
1208
1209static void flush_multipath_work(struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210{
Mike Snitzer518257b2016-03-17 16:32:10 -04001211 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1212 smp_mb__after_atomic();
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +00001213
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001214 flush_workqueue(kmpath_handlerd);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001215 multipath_wait_for_pg_init_completion(m);
Alasdair G Kergona044d012005-07-12 15:53:02 -07001216 flush_workqueue(kmultipathd);
Tejun Heo43829732012-08-20 14:51:24 -07001217 flush_work(&m->trigger_event);
Shiva Krishna Merla954a73d2013-10-30 03:26:38 +00001218
Mike Snitzer518257b2016-03-17 16:32:10 -04001219 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1220 smp_mb__after_atomic();
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001221}
1222
1223static void multipath_dtr(struct dm_target *ti)
1224{
1225 struct multipath *m = ti->private;
1226
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001227 flush_multipath_work(m);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 free_multipath(m);
1229}
1230
1231/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 * Take a path out of use.
1233 */
1234static int fail_path(struct pgpath *pgpath)
1235{
1236 unsigned long flags;
1237 struct multipath *m = pgpath->pg->m;
1238
1239 spin_lock_irqsave(&m->lock, flags);
1240
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001241 if (!pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 goto out;
1243
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001244 DMWARN("Failing path %s.", pgpath->path.dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001245
1246 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001247 pgpath->is_active = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248 pgpath->fail_count++;
1249
Mike Snitzer91e968a2016-03-17 17:10:15 -04001250 atomic_dec(&m->nr_valid_paths);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251
1252 if (pgpath == m->current_pgpath)
1253 m->current_pgpath = NULL;
1254
Mike Andersonb15546f2007-10-19 22:48:02 +01001255 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
Mike Snitzer91e968a2016-03-17 17:10:15 -04001256 pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
Mike Andersonb15546f2007-10-19 22:48:02 +01001257
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001258 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259
1260out:
1261 spin_unlock_irqrestore(&m->lock, flags);
1262
1263 return 0;
1264}
1265
1266/*
1267 * Reinstate a previously-failed path
1268 */
1269static int reinstate_path(struct pgpath *pgpath)
1270{
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001271 int r = 0, run_queue = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272 unsigned long flags;
1273 struct multipath *m = pgpath->pg->m;
Mike Snitzer91e968a2016-03-17 17:10:15 -04001274 unsigned nr_valid_paths;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275
1276 spin_lock_irqsave(&m->lock, flags);
1277
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001278 if (pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279 goto out;
1280
Mike Snitzerec31f3f2016-02-20 12:49:43 -05001281 DMWARN("Reinstating path %s.", pgpath->path.dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282
1283 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1284 if (r)
1285 goto out;
1286
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001287 pgpath->is_active = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288
Mike Snitzer91e968a2016-03-17 17:10:15 -04001289 nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1290 if (nr_valid_paths == 1) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001291 m->current_pgpath = NULL;
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001292 run_queue = 1;
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001293 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001294 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
Mike Snitzer91e968a2016-03-17 17:10:15 -04001295 atomic_inc(&m->pg_init_in_progress);
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001296 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297
Mike Andersonb15546f2007-10-19 22:48:02 +01001298 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
Mike Snitzer91e968a2016-03-17 17:10:15 -04001299 pgpath->path.dev->name, nr_valid_paths);
Mike Andersonb15546f2007-10-19 22:48:02 +01001300
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001301 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001302
1303out:
1304 spin_unlock_irqrestore(&m->lock, flags);
Mike Snitzer76e33fe2016-05-19 16:15:14 -04001305 if (run_queue) {
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001306 dm_table_run_md_queue_async(m->ti->table);
Mike Snitzer76e33fe2016-05-19 16:15:14 -04001307 process_queued_bios_list(m);
1308 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309
1310 return r;
1311}
1312
1313/*
1314 * Fail or reinstate all paths that match the provided struct dm_dev.
1315 */
1316static int action_dev(struct multipath *m, struct dm_dev *dev,
1317 action_fn action)
1318{
Mike Snitzer19040c02011-03-24 13:54:31 +00001319 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320 struct pgpath *pgpath;
1321 struct priority_group *pg;
1322
1323 list_for_each_entry(pg, &m->priority_groups, list) {
1324 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1325 if (pgpath->path.dev == dev)
1326 r = action(pgpath);
1327 }
1328 }
1329
1330 return r;
1331}
1332
1333/*
1334 * Temporarily try to avoid having to use the specified PG
1335 */
1336static void bypass_pg(struct multipath *m, struct priority_group *pg,
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001337 bool bypassed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338{
1339 unsigned long flags;
1340
1341 spin_lock_irqsave(&m->lock, flags);
1342
1343 pg->bypassed = bypassed;
1344 m->current_pgpath = NULL;
1345 m->current_pg = NULL;
1346
1347 spin_unlock_irqrestore(&m->lock, flags);
1348
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001349 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350}
1351
1352/*
1353 * Switch to using the specified PG from the next I/O that gets mapped
1354 */
1355static int switch_pg_num(struct multipath *m, const char *pgstr)
1356{
1357 struct priority_group *pg;
1358 unsigned pgnum;
1359 unsigned long flags;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001360 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001362 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 (pgnum > m->nr_priority_groups)) {
1364 DMWARN("invalid PG number supplied to switch_pg_num");
1365 return -EINVAL;
1366 }
1367
1368 spin_lock_irqsave(&m->lock, flags);
1369 list_for_each_entry(pg, &m->priority_groups, list) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001370 pg->bypassed = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 if (--pgnum)
1372 continue;
1373
1374 m->current_pgpath = NULL;
1375 m->current_pg = NULL;
1376 m->next_pg = pg;
1377 }
1378 spin_unlock_irqrestore(&m->lock, flags);
1379
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001380 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381 return 0;
1382}
1383
1384/*
1385 * Set/clear bypassed status of a PG.
1386 * PGs are numbered upwards from 1 in the order they were declared.
1387 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001388static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389{
1390 struct priority_group *pg;
1391 unsigned pgnum;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001392 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001394 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395 (pgnum > m->nr_priority_groups)) {
1396 DMWARN("invalid PG number supplied to bypass_pg");
1397 return -EINVAL;
1398 }
1399
1400 list_for_each_entry(pg, &m->priority_groups, list) {
1401 if (!--pgnum)
1402 break;
1403 }
1404
1405 bypass_pg(m, pg, bypassed);
1406 return 0;
1407}
1408
1409/*
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001410 * Should we retry pg_init immediately?
1411 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001412static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001413{
1414 unsigned long flags;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001415 bool limit_reached = false;
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001416
1417 spin_lock_irqsave(&m->lock, flags);
1418
Mike Snitzer91e968a2016-03-17 17:10:15 -04001419 if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1420 !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
Mike Snitzer518257b2016-03-17 16:32:10 -04001421 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001422 else
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001423 limit_reached = true;
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001424
1425 spin_unlock_irqrestore(&m->lock, flags);
1426
1427 return limit_reached;
1428}
1429
Chandra Seetharaman3ae31f62009-10-21 09:22:46 -07001430static void pg_init_done(void *data, int errors)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001431{
Moger, Babu83c0d5d2010-03-06 02:29:45 +00001432 struct pgpath *pgpath = data;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001433 struct priority_group *pg = pgpath->pg;
1434 struct multipath *m = pg->m;
1435 unsigned long flags;
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001436 bool delay_retry = false;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001437
1438 /* device or driver problems */
1439 switch (errors) {
1440 case SCSI_DH_OK:
1441 break;
1442 case SCSI_DH_NOSYS:
1443 if (!m->hw_handler_name) {
1444 errors = 0;
1445 break;
1446 }
Moger, Babuf7b934c2010-03-06 02:29:49 +00001447 DMERR("Could not failover the device: Handler scsi_dh_%s "
1448 "Error %d.", m->hw_handler_name, errors);
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001449 /*
1450 * Fail path for now, so we do not ping pong
1451 */
1452 fail_path(pgpath);
1453 break;
1454 case SCSI_DH_DEV_TEMP_BUSY:
1455 /*
1456 * Probably doing something like FW upgrade on the
1457 * controller so try the other pg.
1458 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001459 bypass_pg(m, pg, true);
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001460 break;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001461 case SCSI_DH_RETRY:
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001462 /* Wait before retrying. */
1463 delay_retry = 1;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001464 case SCSI_DH_IMM_RETRY:
1465 case SCSI_DH_RES_TEMP_UNAVAIL:
1466 if (pg_init_limit_reached(m, pgpath))
1467 fail_path(pgpath);
1468 errors = 0;
1469 break;
Mike Snitzerec31f3f2016-02-20 12:49:43 -05001470 case SCSI_DH_DEV_OFFLINED:
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001471 default:
1472 /*
1473 * We probably do not want to fail the path for a device
1474 * error, but this is what the old dm did. In future
1475 * patches we can do more advanced handling.
1476 */
1477 fail_path(pgpath);
1478 }
1479
1480 spin_lock_irqsave(&m->lock, flags);
1481 if (errors) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001482 if (pgpath == m->current_pgpath) {
1483 DMERR("Could not failover device. Error %d.", errors);
1484 m->current_pgpath = NULL;
1485 m->current_pg = NULL;
1486 }
Mike Snitzer518257b2016-03-17 16:32:10 -04001487 } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001488 pg->bypassed = false;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001489
Mike Snitzer91e968a2016-03-17 17:10:15 -04001490 if (atomic_dec_return(&m->pg_init_in_progress) > 0)
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001491 /* Activations of other paths are still on going */
1492 goto out;
1493
Mike Snitzer518257b2016-03-17 16:32:10 -04001494 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1495 if (delay_retry)
1496 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1497 else
1498 clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1499
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001500 if (__pg_init_all_paths(m))
1501 goto out;
1502 }
Mike Snitzer518257b2016-03-17 16:32:10 -04001503 clear_bit(MPATHF_QUEUE_IO, &m->flags);
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001504
Mike Snitzer76e33fe2016-05-19 16:15:14 -04001505 process_queued_bios_list(m);
1506
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001507 /*
1508 * Wake up any thread waiting to suspend.
1509 */
1510 wake_up(&m->pg_init_wait);
1511
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001512out:
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001513 spin_unlock_irqrestore(&m->lock, flags);
1514}
1515
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001516static void activate_path(struct work_struct *work)
1517{
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001518 struct pgpath *pgpath =
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001519 container_of(work, struct pgpath, activate_path.work);
Mike Snitzerf10e06b2016-09-01 12:06:37 -04001520 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001521
Mike Snitzerf10e06b2016-09-01 12:06:37 -04001522 if (pgpath->is_active && !blk_queue_dying(q))
1523 scsi_dh_activate(q, pg_init_done, pgpath);
Hannes Reinecke3a017502014-02-28 15:33:49 +01001524 else
1525 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001526}
1527
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001528static int noretry_error(int error)
1529{
1530 switch (error) {
1531 case -EOPNOTSUPP:
1532 case -EREMOTEIO:
1533 case -EILSEQ:
1534 case -ENODATA:
Jun'ichi Nomuracc9d3c32013-09-13 14:54:30 +09001535 case -ENOSPC:
Hannes Reinecke7e782af2013-07-01 15:16:26 +02001536 return 1;
1537 }
1538
1539 /* Anything else could be a path failure, so should be retried */
1540 return 0;
1541}
1542
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543/*
1544 * end_io handling
1545 */
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001546static int do_end_io(struct multipath *m, struct request *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001547 int error, struct dm_mpath_io *mpio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548{
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001549 /*
1550 * We don't queue any clone request inside the multipath target
1551 * during end I/O handling, since those clone requests don't have
1552 * bio clones. If we queue them inside the multipath target,
1553 * we need to make bio clones, that requires memory allocation.
Mike Snitzer4cc96132016-05-12 16:28:10 -04001554 * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001555 * don't have bio clones.)
1556 * Instead of queueing the clone request here, we queue the original
1557 * request into dm core, which will remake a clone request and
1558 * clone bios for it and resubmit it later.
1559 */
1560 int r = DM_ENDIO_REQUEUE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001562 if (!error && !clone->errors)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563 return 0; /* I/O complete */
1564
Mike Snitzer7eee4ae2014-06-02 15:50:06 -04001565 if (noretry_error(error))
Mike Snitzer959eb4e2010-08-12 04:14:32 +01001566 return error;
1567
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001568 if (mpio->pgpath)
1569 fail_path(mpio->pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570
Mike Snitzer91e968a2016-03-17 17:10:15 -04001571 if (!atomic_read(&m->nr_valid_paths)) {
Mike Snitzer518257b2016-03-17 16:32:10 -04001572 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
Mike Snitzer76e33fe2016-05-19 16:15:14 -04001573 if (!must_push_back_rq(m))
Hannes Reinecke751b2a72011-01-18 10:13:12 +01001574 r = -EIO;
1575 } else {
1576 if (error == -EBADE)
1577 r = error;
1578 }
1579 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001581 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001582}
1583
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001584static int multipath_end_io(struct dm_target *ti, struct request *clone,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585 int error, union map_info *map_context)
1586{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001587 struct multipath *m = ti->private;
Mike Snitzer2eff1922016-02-03 09:13:14 -05001588 struct dm_mpath_io *mpio = get_mpio(map_context);
Wei Yongjuna71a2612012-10-12 16:59:42 +01001589 struct pgpath *pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590 struct path_selector *ps;
1591 int r;
1592
Jun'ichi Nomura466891f2012-03-28 18:41:25 +01001593 BUG_ON(!mpio);
1594
Mike Snitzer2eff1922016-02-03 09:13:14 -05001595 r = do_end_io(m, clone, error, mpio);
Wei Yongjuna71a2612012-10-12 16:59:42 +01001596 pgpath = mpio->pgpath;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597 if (pgpath) {
1598 ps = &pgpath->pg->ps;
1599 if (ps->type->end_io)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001600 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 }
Mike Snitzer2eff1922016-02-03 09:13:14 -05001602 clear_request_fn_mpio(m, map_context);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603
1604 return r;
1605}
1606
Mike Snitzer76e33fe2016-05-19 16:15:14 -04001607static int do_end_io_bio(struct multipath *m, struct bio *clone,
1608 int error, struct dm_mpath_io *mpio)
1609{
1610 unsigned long flags;
1611
1612 if (!error)
1613 return 0; /* I/O complete */
1614
1615 if (noretry_error(error))
1616 return error;
1617
1618 if (mpio->pgpath)
1619 fail_path(mpio->pgpath);
1620
1621 if (!atomic_read(&m->nr_valid_paths)) {
1622 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1623 if (!must_push_back_bio(m))
1624 return -EIO;
1625 return DM_ENDIO_REQUEUE;
1626 } else {
1627 if (error == -EBADE)
1628 return error;
1629 }
1630 }
1631
1632 /* Queue for the daemon to resubmit */
Mike Snitzerbf661be2016-05-24 15:48:08 -04001633 dm_bio_restore(get_bio_details_from_bio(clone), clone);
Mike Snitzer76e33fe2016-05-19 16:15:14 -04001634
1635 spin_lock_irqsave(&m->lock, flags);
1636 bio_list_add(&m->queued_bios, clone);
1637 spin_unlock_irqrestore(&m->lock, flags);
1638 if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1639 queue_work(kmultipathd, &m->process_queued_bios);
1640
1641 return DM_ENDIO_INCOMPLETE;
1642}
1643
1644static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, int error)
1645{
1646 struct multipath *m = ti->private;
1647 struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1648 struct pgpath *pgpath;
1649 struct path_selector *ps;
1650 int r;
1651
1652 BUG_ON(!mpio);
1653
1654 r = do_end_io_bio(m, clone, error, mpio);
1655 pgpath = mpio->pgpath;
1656 if (pgpath) {
1657 ps = &pgpath->pg->ps;
1658 if (ps->type->end_io)
1659 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1660 }
1661
1662 return r;
1663}
1664
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665/*
1666 * Suspend can't complete until all the I/O is processed so if
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001667 * the last path fails we must error any remaining I/O.
1668 * Note that if the freeze_bdev fails while suspending, the
1669 * queue_if_no_path state is lost - userspace should reset it.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670 */
1671static void multipath_presuspend(struct dm_target *ti)
1672{
Mike Snitzer7943bd62016-02-02 21:53:15 -05001673 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001675 queue_if_no_path(m, false, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676}
1677
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001678static void multipath_postsuspend(struct dm_target *ti)
1679{
Mike Anderson6380f262009-12-10 23:52:21 +00001680 struct multipath *m = ti->private;
1681
1682 mutex_lock(&m->work_mutex);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001683 flush_multipath_work(m);
Mike Anderson6380f262009-12-10 23:52:21 +00001684 mutex_unlock(&m->work_mutex);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001685}
1686
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001687/*
1688 * Restore the queue_if_no_path setting.
1689 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690static void multipath_resume(struct dm_target *ti)
1691{
Mike Snitzer7943bd62016-02-02 21:53:15 -05001692 struct multipath *m = ti->private;
Mike Snitzer1814f2e2016-07-25 21:08:51 -04001693 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694
Mike Snitzer1814f2e2016-07-25 21:08:51 -04001695 spin_lock_irqsave(&m->lock, flags);
Mike Snitzer518257b2016-03-17 16:32:10 -04001696 if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags))
1697 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
1698 else
1699 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
Mike Snitzer1814f2e2016-07-25 21:08:51 -04001700 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701}
1702
1703/*
1704 * Info output has the following format:
1705 * num_multipath_feature_args [multipath_feature_args]*
1706 * num_handler_status_args [handler_status_args]*
1707 * num_groups init_group_number
1708 * [A|D|E num_ps_status_args [ps_status_args]*
1709 * num_paths num_selector_args
1710 * [path_dev A|F fail_count [selector_args]* ]+ ]+
1711 *
1712 * Table output has the following format (identical to the constructor string):
1713 * num_feature_args [features_args]*
1714 * num_handler_args hw_handler [hw_handler_args]*
1715 * num_groups init_group_number
1716 * [priority selector-name num_ps_args [ps_args]*
1717 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1718 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00001719static void multipath_status(struct dm_target *ti, status_type_t type,
1720 unsigned status_flags, char *result, unsigned maxlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721{
1722 int sz = 0;
1723 unsigned long flags;
Mike Snitzer7943bd62016-02-02 21:53:15 -05001724 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725 struct priority_group *pg;
1726 struct pgpath *p;
1727 unsigned pg_num;
1728 char state;
1729
1730 spin_lock_irqsave(&m->lock, flags);
1731
1732 /* Features */
1733 if (type == STATUSTYPE_INFO)
Mike Snitzer91e968a2016-03-17 17:10:15 -04001734 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1735 atomic_read(&m->pg_init_count));
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001736 else {
Mike Snitzer518257b2016-03-17 16:32:10 -04001737 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001738 (m->pg_init_retries > 0) * 2 +
Mike Snitzera58a9352012-07-27 15:08:04 +01001739 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
Mike Snitzere83068a2016-05-24 21:16:51 -04001740 test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1741 (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1742
Mike Snitzer518257b2016-03-17 16:32:10 -04001743 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001744 DMEMIT("queue_if_no_path ");
1745 if (m->pg_init_retries)
1746 DMEMIT("pg_init_retries %u ", m->pg_init_retries);
Chandra Seetharaman4e2d19e2011-01-13 20:00:01 +00001747 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1748 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
Mike Snitzer518257b2016-03-17 16:32:10 -04001749 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
Mike Snitzera58a9352012-07-27 15:08:04 +01001750 DMEMIT("retain_attached_hw_handler ");
Mike Snitzere83068a2016-05-24 21:16:51 -04001751 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1752 switch(m->queue_mode) {
1753 case DM_TYPE_BIO_BASED:
1754 DMEMIT("queue_mode bio ");
1755 break;
1756 case DM_TYPE_MQ_REQUEST_BASED:
1757 DMEMIT("queue_mode mq ");
1758 break;
1759 }
1760 }
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001761 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001763 if (!m->hw_handler_name || type == STATUSTYPE_INFO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764 DMEMIT("0 ");
1765 else
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001766 DMEMIT("1 %s ", m->hw_handler_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001767
1768 DMEMIT("%u ", m->nr_priority_groups);
1769
1770 if (m->next_pg)
1771 pg_num = m->next_pg->pg_num;
1772 else if (m->current_pg)
1773 pg_num = m->current_pg->pg_num;
1774 else
Mike Snitzera490a072011-03-24 13:54:33 +00001775 pg_num = (m->nr_priority_groups ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776
1777 DMEMIT("%u ", pg_num);
1778
1779 switch (type) {
1780 case STATUSTYPE_INFO:
1781 list_for_each_entry(pg, &m->priority_groups, list) {
1782 if (pg->bypassed)
1783 state = 'D'; /* Disabled */
1784 else if (pg == m->current_pg)
1785 state = 'A'; /* Currently Active */
1786 else
1787 state = 'E'; /* Enabled */
1788
1789 DMEMIT("%c ", state);
1790
1791 if (pg->ps.type->status)
1792 sz += pg->ps.type->status(&pg->ps, NULL, type,
1793 result + sz,
1794 maxlen - sz);
1795 else
1796 DMEMIT("0 ");
1797
1798 DMEMIT("%u %u ", pg->nr_pgpaths,
1799 pg->ps.type->info_args);
1800
1801 list_for_each_entry(p, &pg->pgpaths, list) {
1802 DMEMIT("%s %s %u ", p->path.dev->name,
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001803 p->is_active ? "A" : "F",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001804 p->fail_count);
1805 if (pg->ps.type->status)
1806 sz += pg->ps.type->status(&pg->ps,
1807 &p->path, type, result + sz,
1808 maxlen - sz);
1809 }
1810 }
1811 break;
1812
1813 case STATUSTYPE_TABLE:
1814 list_for_each_entry(pg, &m->priority_groups, list) {
1815 DMEMIT("%s ", pg->ps.type->name);
1816
1817 if (pg->ps.type->status)
1818 sz += pg->ps.type->status(&pg->ps, NULL, type,
1819 result + sz,
1820 maxlen - sz);
1821 else
1822 DMEMIT("0 ");
1823
1824 DMEMIT("%u %u ", pg->nr_pgpaths,
1825 pg->ps.type->table_args);
1826
1827 list_for_each_entry(p, &pg->pgpaths, list) {
1828 DMEMIT("%s ", p->path.dev->name);
1829 if (pg->ps.type->status)
1830 sz += pg->ps.type->status(&pg->ps,
1831 &p->path, type, result + sz,
1832 maxlen - sz);
1833 }
1834 }
1835 break;
1836 }
1837
1838 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839}
1840
1841static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1842{
Mike Anderson6380f262009-12-10 23:52:21 +00001843 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844 struct dm_dev *dev;
Mike Snitzer7943bd62016-02-02 21:53:15 -05001845 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001846 action_fn action;
1847
Mike Anderson6380f262009-12-10 23:52:21 +00001848 mutex_lock(&m->work_mutex);
1849
Kiyoshi Uedac2f3d242009-12-10 23:52:27 +00001850 if (dm_suspended(ti)) {
1851 r = -EBUSY;
1852 goto out;
1853 }
1854
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855 if (argc == 1) {
Mike Snitzer498f0102011-08-02 12:32:04 +01001856 if (!strcasecmp(argv[0], "queue_if_no_path")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001857 r = queue_if_no_path(m, true, false);
Mike Anderson6380f262009-12-10 23:52:21 +00001858 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001859 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001860 r = queue_if_no_path(m, false, false);
Mike Anderson6380f262009-12-10 23:52:21 +00001861 goto out;
1862 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 }
1864
Mike Anderson6380f262009-12-10 23:52:21 +00001865 if (argc != 2) {
Jose Castilloa356e422014-01-29 17:52:45 +01001866 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
Mike Anderson6380f262009-12-10 23:52:21 +00001867 goto out;
1868 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869
Mike Snitzer498f0102011-08-02 12:32:04 +01001870 if (!strcasecmp(argv[0], "disable_group")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001871 r = bypass_pg_num(m, argv[1], true);
Mike Anderson6380f262009-12-10 23:52:21 +00001872 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001873 } else if (!strcasecmp(argv[0], "enable_group")) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001874 r = bypass_pg_num(m, argv[1], false);
Mike Anderson6380f262009-12-10 23:52:21 +00001875 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001876 } else if (!strcasecmp(argv[0], "switch_group")) {
Mike Anderson6380f262009-12-10 23:52:21 +00001877 r = switch_pg_num(m, argv[1]);
1878 goto out;
Mike Snitzer498f0102011-08-02 12:32:04 +01001879 } else if (!strcasecmp(argv[0], "reinstate_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001880 action = reinstate_path;
Mike Snitzer498f0102011-08-02 12:32:04 +01001881 else if (!strcasecmp(argv[0], "fail_path"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882 action = fail_path;
Mike Anderson6380f262009-12-10 23:52:21 +00001883 else {
Jose Castilloa356e422014-01-29 17:52:45 +01001884 DMWARN("Unrecognised multipath message received: %s", argv[0]);
Mike Anderson6380f262009-12-10 23:52:21 +00001885 goto out;
1886 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +00001888 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001889 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001890 DMWARN("message: error getting device %s",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001891 argv[1]);
Mike Anderson6380f262009-12-10 23:52:21 +00001892 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001893 }
1894
1895 r = action_dev(m, dev, action);
1896
1897 dm_put_device(ti, dev);
1898
Mike Anderson6380f262009-12-10 23:52:21 +00001899out:
1900 mutex_unlock(&m->work_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902}
1903
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001904static int multipath_prepare_ioctl(struct dm_target *ti,
1905 struct block_device **bdev, fmode_t *mode)
Milan Broz9af4aa32006-10-03 01:15:20 -07001906{
Mikulas Patocka35991652012-06-03 00:29:58 +01001907 struct multipath *m = ti->private;
Mike Snitzer2da16102016-03-17 18:38:17 -04001908 struct pgpath *current_pgpath;
Mikulas Patocka35991652012-06-03 00:29:58 +01001909 int r;
1910
Mike Snitzer2da16102016-03-17 18:38:17 -04001911 current_pgpath = lockless_dereference(m->current_pgpath);
1912 if (!current_pgpath)
1913 current_pgpath = choose_pgpath(m, 0);
Milan Broz9af4aa32006-10-03 01:15:20 -07001914
Mike Snitzer2da16102016-03-17 18:38:17 -04001915 if (current_pgpath) {
Mike Snitzer518257b2016-03-17 16:32:10 -04001916 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
Mike Snitzer2da16102016-03-17 18:38:17 -04001917 *bdev = current_pgpath->path.dev->bdev;
1918 *mode = current_pgpath->path.dev->mode;
Junichi Nomura43e43c92015-11-17 09:36:56 +00001919 r = 0;
1920 } else {
1921 /* pg_init has not started or completed */
1922 r = -ENOTCONN;
1923 }
1924 } else {
1925 /* No path is available */
Mike Snitzer518257b2016-03-17 16:32:10 -04001926 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
Junichi Nomura43e43c92015-11-17 09:36:56 +00001927 r = -ENOTCONN;
1928 else
1929 r = -EIO;
Milan Broze90dae12006-10-03 01:15:22 -07001930 }
Milan Broz9af4aa32006-10-03 01:15:20 -07001931
Junichi Nomura5bbbfdf2015-11-17 09:39:26 +00001932 if (r == -ENOTCONN) {
Mike Snitzer2da16102016-03-17 18:38:17 -04001933 if (!lockless_dereference(m->current_pg)) {
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001934 /* Path status changed, redo selection */
Mike Snitzer2da16102016-03-17 18:38:17 -04001935 (void) choose_pgpath(m, 0);
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001936 }
Mike Snitzer518257b2016-03-17 16:32:10 -04001937 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
Mike Snitzer2da16102016-03-17 18:38:17 -04001938 pg_init_all_paths(m);
Hannes Reinecke63d832c2014-05-26 14:45:39 +02001939 dm_table_run_md_queue_async(m->ti->table);
Mike Snitzer76e33fe2016-05-19 16:15:14 -04001940 process_queued_bios_list(m);
Hannes Reinecke3e9f1be2014-02-28 15:33:45 +01001941 }
Mikulas Patocka35991652012-06-03 00:29:58 +01001942
Christoph Hellwige56f81e2015-10-15 14:10:50 +02001943 /*
1944 * Only pass ioctls through if the device sizes match exactly.
1945 */
1946 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1947 return 1;
1948 return r;
Milan Broz9af4aa32006-10-03 01:15:20 -07001949}
1950
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001951static int multipath_iterate_devices(struct dm_target *ti,
1952 iterate_devices_callout_fn fn, void *data)
1953{
1954 struct multipath *m = ti->private;
1955 struct priority_group *pg;
1956 struct pgpath *p;
1957 int ret = 0;
1958
1959 list_for_each_entry(pg, &m->priority_groups, list) {
1960 list_for_each_entry(p, &pg->pgpaths, list) {
Mike Snitzer5dea2712009-07-23 20:30:42 +01001961 ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001962 if (ret)
1963 goto out;
1964 }
1965 }
1966
1967out:
1968 return ret;
1969}
1970
Mike Snitzer9f54cec2016-02-11 21:42:28 -05001971static int pgpath_busy(struct pgpath *pgpath)
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001972{
1973 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1974
Mike Snitzer52b09912015-02-23 16:36:41 -05001975 return blk_lld_busy(q);
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001976}
1977
1978/*
1979 * We return "busy", only when we can map I/Os but underlying devices
1980 * are busy (so even if we map I/Os now, the I/Os will wait on
1981 * the underlying queue).
1982 * In other words, if we want to kill I/Os or queue them inside us
1983 * due to map unavailability, we don't return "busy". Otherwise,
1984 * dm core won't give us the I/Os and we can't do what we want.
1985 */
1986static int multipath_busy(struct dm_target *ti)
1987{
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05001988 bool busy = false, has_active = false;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001989 struct multipath *m = ti->private;
Mike Snitzer2da16102016-03-17 18:38:17 -04001990 struct priority_group *pg, *next_pg;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001991 struct pgpath *pgpath;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001992
Jun'ichi Nomura7a7a3b42014-07-08 00:55:14 +00001993 /* pg_init in progress or no paths available */
Mike Snitzer91e968a2016-03-17 17:10:15 -04001994 if (atomic_read(&m->pg_init_in_progress) ||
Mike Snitzer2da16102016-03-17 18:38:17 -04001995 (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)))
1996 return true;
1997
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001998 /* Guess which priority_group will be used at next mapping time */
Mike Snitzer2da16102016-03-17 18:38:17 -04001999 pg = lockless_dereference(m->current_pg);
2000 next_pg = lockless_dereference(m->next_pg);
2001 if (unlikely(!lockless_dereference(m->current_pgpath) && next_pg))
2002 pg = next_pg;
2003
2004 if (!pg) {
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002005 /*
2006 * We don't know which pg will be used at next mapping time.
Mike Snitzer2da16102016-03-17 18:38:17 -04002007 * We don't call choose_pgpath() here to avoid to trigger
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002008 * pg_init just by busy checking.
2009 * So we don't know whether underlying devices we will be using
2010 * at next mapping time are busy or not. Just try mapping.
2011 */
Mike Snitzer2da16102016-03-17 18:38:17 -04002012 return busy;
2013 }
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002014
2015 /*
2016 * If there is one non-busy active path at least, the path selector
2017 * will be able to select it. So we consider such a pg as not busy.
2018 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05002019 busy = true;
Mike Snitzer2da16102016-03-17 18:38:17 -04002020 list_for_each_entry(pgpath, &pg->pgpaths, list) {
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002021 if (pgpath->is_active) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05002022 has_active = true;
Mike Snitzer9f54cec2016-02-11 21:42:28 -05002023 if (!pgpath_busy(pgpath)) {
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05002024 busy = false;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002025 break;
2026 }
2027 }
Mike Snitzer2da16102016-03-17 18:38:17 -04002028 }
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002029
Mike Snitzer2da16102016-03-17 18:38:17 -04002030 if (!has_active) {
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002031 /*
2032 * No active path in this pg, so this pg won't be used and
2033 * the current_pg will be changed at next mapping time.
2034 * We need to try mapping to determine it.
2035 */
Mike Snitzerbe7d31c2016-02-10 13:02:21 -05002036 busy = false;
Mike Snitzer2da16102016-03-17 18:38:17 -04002037 }
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002038
2039 return busy;
2040}
2041
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042/*-----------------------------------------------------------------
2043 * Module setup
2044 *---------------------------------------------------------------*/
2045static struct target_type multipath_target = {
2046 .name = "multipath",
Mike Snitzere83068a2016-05-24 21:16:51 -04002047 .version = {1, 12, 0},
Mike Snitzer16f12262016-01-31 17:22:27 -05002048 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049 .module = THIS_MODULE,
2050 .ctr = multipath_ctr,
2051 .dtr = multipath_dtr,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002052 .map_rq = multipath_map,
Mike Snitzere5863d92014-12-17 21:08:12 -05002053 .clone_and_map_rq = multipath_clone_and_map,
2054 .release_clone_rq = multipath_release_clone,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01002055 .rq_end_io = multipath_end_io,
Mike Snitzer76e33fe2016-05-19 16:15:14 -04002056 .map = multipath_map_bio,
2057 .end_io = multipath_end_io_bio,
2058 .presuspend = multipath_presuspend,
2059 .postsuspend = multipath_postsuspend,
2060 .resume = multipath_resume,
2061 .status = multipath_status,
2062 .message = multipath_message,
2063 .prepare_ioctl = multipath_prepare_ioctl,
2064 .iterate_devices = multipath_iterate_devices,
2065 .busy = multipath_busy,
2066};
2067
Linus Torvalds1da177e2005-04-16 15:20:36 -07002068static int __init dm_multipath_init(void)
2069{
2070 int r;
2071
Mike Snitzer76e33fe2016-05-19 16:15:14 -04002072 /* allocate a slab for the dm_mpath_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +01002073 _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074 if (!_mpio_cache)
2075 return -ENOMEM;
2076
2077 r = dm_register_target(&multipath_target);
2078 if (r < 0) {
Mike Snitzer76e33fe2016-05-19 16:15:14 -04002079 DMERR("request-based register failed %d", r);
Johannes Thumshirnff658e92015-01-11 12:45:23 +01002080 r = -EINVAL;
2081 goto bad_register_target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002082 }
2083
Tejun Heo4d4d66a2011-01-13 19:59:57 +00002084 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07002085 if (!kmultipathd) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01002086 DMERR("failed to create workqueue kmpathd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01002087 r = -ENOMEM;
2088 goto bad_alloc_kmultipathd;
Alasdair G Kergonc5573082005-05-05 16:16:07 -07002089 }
2090
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07002091 /*
2092 * A separate workqueue is used to handle the device handlers
2093 * to avoid overloading existing workqueue. Overloading the
2094 * old workqueue would also create a bottleneck in the
2095 * path of the storage hardware device activation.
2096 */
Tejun Heo4d4d66a2011-01-13 19:59:57 +00002097 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2098 WQ_MEM_RECLAIM);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07002099 if (!kmpath_handlerd) {
2100 DMERR("failed to create workqueue kmpath_handlerd");
Johannes Thumshirnff658e92015-01-11 12:45:23 +01002101 r = -ENOMEM;
2102 goto bad_alloc_kmpath_handlerd;
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07002103 }
2104
Johannes Thumshirnff658e92015-01-11 12:45:23 +01002105 return 0;
2106
2107bad_alloc_kmpath_handlerd:
2108 destroy_workqueue(kmultipathd);
2109bad_alloc_kmultipathd:
2110 dm_unregister_target(&multipath_target);
2111bad_register_target:
2112 kmem_cache_destroy(_mpio_cache);
2113
Linus Torvalds1da177e2005-04-16 15:20:36 -07002114 return r;
2115}
2116
2117static void __exit dm_multipath_exit(void)
2118{
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07002119 destroy_workqueue(kmpath_handlerd);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07002120 destroy_workqueue(kmultipathd);
2121
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00002122 dm_unregister_target(&multipath_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123 kmem_cache_destroy(_mpio_cache);
2124}
2125
Linus Torvalds1da177e2005-04-16 15:20:36 -07002126module_init(dm_multipath_init);
2127module_exit(dm_multipath_exit);
2128
2129MODULE_DESCRIPTION(DM_NAME " multipath target");
2130MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2131MODULE_LICENSE("GPL");