blob: c1335487cc724c39c1b2c4940b65b5bb0d8e5baf [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
Mikulas Patocka586e80e2008-10-21 17:44:59 +01008#include <linux/device-mapper.h>
9
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include "dm-path-selector.h"
Mike Andersonb15546f2007-10-19 22:48:02 +010011#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070012
13#include <linux/ctype.h>
14#include <linux/init.h>
15#include <linux/mempool.h>
16#include <linux/module.h>
17#include <linux/pagemap.h>
18#include <linux/slab.h>
19#include <linux/time.h>
20#include <linux/workqueue.h>
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070021#include <scsi/scsi_dh.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <asm/atomic.h>
23
Alasdair G Kergon72d94862006-06-26 00:27:35 -070024#define DM_MSG_PREFIX "multipath"
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#define MESG_STR(x) x, sizeof(x)
26
27/* Path properties */
28struct pgpath {
29 struct list_head list;
30
31 struct priority_group *pg; /* Owning PG */
Kiyoshi Ueda66800732008-10-10 13:36:58 +010032 unsigned is_active; /* Path status */
Linus Torvalds1da177e2005-04-16 15:20:36 -070033 unsigned fail_count; /* Cumulative failure count */
34
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -080035 struct dm_path path;
Mike Anderson224cb3e2008-08-29 09:36:09 +020036 struct work_struct deactivate_path;
Chandra Seetharamane54f77d2009-06-22 10:12:12 +010037 struct work_struct activate_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -070038};
39
40#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
41
42/*
43 * Paths are grouped into Priority Groups and numbered from 1 upwards.
44 * Each has a path selector which controls which path gets used.
45 */
46struct priority_group {
47 struct list_head list;
48
49 struct multipath *m; /* Owning multipath instance */
50 struct path_selector ps;
51
52 unsigned pg_num; /* Reference number */
53 unsigned bypassed; /* Temporarily bypass this PG? */
54
55 unsigned nr_pgpaths; /* Number of paths in PG */
56 struct list_head pgpaths;
57};
58
59/* Multipath context */
60struct multipath {
61 struct list_head list;
62 struct dm_target *ti;
63
64 spinlock_t lock;
65
Chandra Seetharamancfae5c92008-05-01 14:50:11 -070066 const char *hw_handler_name;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -070067 char *hw_handler_params;
Linus Torvalds1da177e2005-04-16 15:20:36 -070068 unsigned nr_priority_groups;
69 struct list_head priority_groups;
70 unsigned pg_init_required; /* pg_init needs calling? */
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -070071 unsigned pg_init_in_progress; /* Only one pg_init allowed at once */
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +000072 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
74 unsigned nr_valid_paths; /* Total number of usable paths */
75 struct pgpath *current_pgpath;
76 struct priority_group *current_pg;
77 struct priority_group *next_pg; /* Switch to this PG if set */
78 unsigned repeat_count; /* I/Os left before calling PS again */
79
80 unsigned queue_io; /* Must we queue all I/O? */
81 unsigned queue_if_no_path; /* Queue I/O if last path fails? */
Alasdair G Kergon436d4102005-07-12 15:53:03 -070082 unsigned saved_queue_if_no_path;/* Saved state during suspension */
Dave Wysochanskic9e45582007-10-19 22:47:53 +010083 unsigned pg_init_retries; /* Number of times to retry pg_init */
84 unsigned pg_init_count; /* Number of times pg_init called */
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
86 struct work_struct process_queued_ios;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +010087 struct list_head queued_ios;
Linus Torvalds1da177e2005-04-16 15:20:36 -070088 unsigned queue_size;
89
90 struct work_struct trigger_event;
91
92 /*
Alasdair G Kergon028867a2007-07-12 17:26:32 +010093 * We must use a mempool of dm_mpath_io structs so that we
Linus Torvalds1da177e2005-04-16 15:20:36 -070094 * can resubmit bios on error.
95 */
96 mempool_t *mpio_pool;
Mike Anderson6380f262009-12-10 23:52:21 +000097
98 struct mutex work_mutex;
Linus Torvalds1da177e2005-04-16 15:20:36 -070099};
100
101/*
102 * Context information attached to each bio we process.
103 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100104struct dm_mpath_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 struct pgpath *pgpath;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100106 size_t nr_bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107};
108
109typedef int (*action_fn) (struct pgpath *pgpath);
110
111#define MIN_IOS 256 /* Mempool size */
112
Christoph Lametere18b8902006-12-06 20:33:20 -0800113static struct kmem_cache *_mpio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700115static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
David Howellsc4028952006-11-22 14:57:56 +0000116static void process_queued_ios(struct work_struct *work);
117static void trigger_event(struct work_struct *work);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700118static void activate_path(struct work_struct *work);
Mike Anderson224cb3e2008-08-29 09:36:09 +0200119static void deactivate_path(struct work_struct *work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120
121
122/*-----------------------------------------------
123 * Allocation routines
124 *-----------------------------------------------*/
125
126static struct pgpath *alloc_pgpath(void)
127{
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700128 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
Mike Anderson224cb3e2008-08-29 09:36:09 +0200130 if (pgpath) {
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100131 pgpath->is_active = 1;
Mike Anderson224cb3e2008-08-29 09:36:09 +0200132 INIT_WORK(&pgpath->deactivate_path, deactivate_path);
Chandra Seetharamane54f77d2009-06-22 10:12:12 +0100133 INIT_WORK(&pgpath->activate_path, activate_path);
Mike Anderson224cb3e2008-08-29 09:36:09 +0200134 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135
136 return pgpath;
137}
138
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100139static void free_pgpath(struct pgpath *pgpath)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140{
141 kfree(pgpath);
142}
143
Mike Anderson224cb3e2008-08-29 09:36:09 +0200144static void deactivate_path(struct work_struct *work)
145{
146 struct pgpath *pgpath =
147 container_of(work, struct pgpath, deactivate_path);
148
149 blk_abort_queue(pgpath->path.dev->bdev->bd_disk->queue);
150}
151
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152static struct priority_group *alloc_priority_group(void)
153{
154 struct priority_group *pg;
155
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700156 pg = kzalloc(sizeof(*pg), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700158 if (pg)
159 INIT_LIST_HEAD(&pg->pgpaths);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160
161 return pg;
162}
163
164static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
165{
166 struct pgpath *pgpath, *tmp;
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700167 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
169 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
170 list_del(&pgpath->list);
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700171 if (m->hw_handler_name)
172 scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 dm_put_device(ti, pgpath->path.dev);
174 free_pgpath(pgpath);
175 }
176}
177
178static void free_priority_group(struct priority_group *pg,
179 struct dm_target *ti)
180{
181 struct path_selector *ps = &pg->ps;
182
183 if (ps->type) {
184 ps->type->destroy(ps);
185 dm_put_path_selector(ps->type);
186 }
187
188 free_pgpaths(&pg->pgpaths, ti);
189 kfree(pg);
190}
191
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700192static struct multipath *alloc_multipath(struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193{
194 struct multipath *m;
195
Micha³ Miros³awe69fae52006-10-03 01:15:34 -0700196 m = kzalloc(sizeof(*m), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 if (m) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 INIT_LIST_HEAD(&m->priority_groups);
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100199 INIT_LIST_HEAD(&m->queued_ios);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 spin_lock_init(&m->lock);
201 m->queue_io = 1;
David Howellsc4028952006-11-22 14:57:56 +0000202 INIT_WORK(&m->process_queued_ios, process_queued_ios);
203 INIT_WORK(&m->trigger_event, trigger_event);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000204 init_waitqueue_head(&m->pg_init_wait);
Mike Anderson6380f262009-12-10 23:52:21 +0000205 mutex_init(&m->work_mutex);
Matthew Dobson93d23412006-03-26 01:37:50 -0800206 m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 if (!m->mpio_pool) {
208 kfree(m);
209 return NULL;
210 }
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700211 m->ti = ti;
212 ti->private = m;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 }
214
215 return m;
216}
217
218static void free_multipath(struct multipath *m)
219{
220 struct priority_group *pg, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221
222 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
223 list_del(&pg->list);
224 free_priority_group(pg, m->ti);
225 }
226
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700227 kfree(m->hw_handler_name);
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700228 kfree(m->hw_handler_params);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 mempool_destroy(m->mpio_pool);
230 kfree(m);
231}
232
233
234/*-----------------------------------------------
235 * Path selection
236 *-----------------------------------------------*/
237
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000238static void __pg_init_all_paths(struct multipath *m)
239{
240 struct pgpath *pgpath;
241
242 m->pg_init_count++;
243 m->pg_init_required = 0;
244 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
245 /* Skip failed paths */
246 if (!pgpath->is_active)
247 continue;
248 if (queue_work(kmpath_handlerd, &pgpath->activate_path))
249 m->pg_init_in_progress++;
250 }
251}
252
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
254{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 m->current_pg = pgpath->pg;
256
257 /* Must we initialise the PG first, and queue I/O till it's ready? */
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700258 if (m->hw_handler_name) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 m->pg_init_required = 1;
260 m->queue_io = 1;
261 } else {
262 m->pg_init_required = 0;
263 m->queue_io = 0;
264 }
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100265
266 m->pg_init_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267}
268
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100269static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
270 size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271{
Josef "Jeff" Sipekc922d5f2006-12-08 02:36:33 -0800272 struct dm_path *path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100274 path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 if (!path)
276 return -ENXIO;
277
278 m->current_pgpath = path_to_pgpath(path);
279
280 if (m->current_pg != pg)
281 __switch_pg(m, m->current_pgpath);
282
283 return 0;
284}
285
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100286static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287{
288 struct priority_group *pg;
289 unsigned bypassed = 1;
290
291 if (!m->nr_valid_paths)
292 goto failed;
293
294 /* Were we instructed to switch PG? */
295 if (m->next_pg) {
296 pg = m->next_pg;
297 m->next_pg = NULL;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100298 if (!__choose_path_in_pg(m, pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 return;
300 }
301
302 /* Don't change PG until it has no remaining paths */
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100303 if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 return;
305
306 /*
307 * Loop through priority groups until we find a valid path.
308 * First time we skip PGs marked 'bypassed'.
309 * Second time we only try the ones we skipped.
310 */
311 do {
312 list_for_each_entry(pg, &m->priority_groups, list) {
313 if (pg->bypassed == bypassed)
314 continue;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100315 if (!__choose_path_in_pg(m, pg, nr_bytes))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 return;
317 }
318 } while (bypassed--);
319
320failed:
321 m->current_pgpath = NULL;
322 m->current_pg = NULL;
323}
324
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800325/*
326 * Check whether bios must be queued in the device-mapper core rather
327 * than here in the target.
328 *
329 * m->lock must be held on entry.
330 *
331 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
332 * same value then we are not between multipath_presuspend()
333 * and multipath_resume() calls and we have no need to check
334 * for the DMF_NOFLUSH_SUSPENDING flag.
335 */
336static int __must_push_back(struct multipath *m)
337{
338 return (m->queue_if_no_path != m->saved_queue_if_no_path &&
339 dm_noflush_suspending(m->ti));
340}
341
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100342static int map_io(struct multipath *m, struct request *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100343 struct dm_mpath_io *mpio, unsigned was_queued)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344{
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -0800345 int r = DM_MAPIO_REMAPPED;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100346 size_t nr_bytes = blk_rq_bytes(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 unsigned long flags;
348 struct pgpath *pgpath;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100349 struct block_device *bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
351 spin_lock_irqsave(&m->lock, flags);
352
353 /* Do we need to select a new pgpath? */
354 if (!m->current_pgpath ||
355 (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100356 __choose_pgpath(m, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357
358 pgpath = m->current_pgpath;
359
360 if (was_queued)
361 m->queue_size--;
362
363 if ((pgpath && m->queue_io) ||
Alasdair G Kergon436d4102005-07-12 15:53:03 -0700364 (!pgpath && m->queue_if_no_path)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 /* Queue for the daemon to resubmit */
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100366 list_add_tail(&clone->queuelist, &m->queued_ios);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 m->queue_size++;
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -0700368 if ((m->pg_init_required && !m->pg_init_in_progress) ||
369 !m->queue_io)
Alasdair G Kergonc5573082005-05-05 16:16:07 -0700370 queue_work(kmultipathd, &m->process_queued_ios);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 pgpath = NULL;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -0800372 r = DM_MAPIO_SUBMITTED;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100373 } else if (pgpath) {
374 bdev = pgpath->path.dev->bdev;
375 clone->q = bdev_get_queue(bdev);
376 clone->rq_disk = bdev->bd_disk;
377 } else if (__must_push_back(m))
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800378 r = DM_MAPIO_REQUEUE;
379 else
380 r = -EIO; /* Failed */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381
382 mpio->pgpath = pgpath;
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100383 mpio->nr_bytes = nr_bytes;
384
385 if (r == DM_MAPIO_REMAPPED && pgpath->pg->ps.type->start_io)
386 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, &pgpath->path,
387 nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388
389 spin_unlock_irqrestore(&m->lock, flags);
390
391 return r;
392}
393
394/*
395 * If we run out of usable paths, should we queue I/O or error it?
396 */
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700397static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
398 unsigned save_old_value)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399{
400 unsigned long flags;
401
402 spin_lock_irqsave(&m->lock, flags);
403
Alasdair G Kergon485ef692005-09-27 21:45:45 -0700404 if (save_old_value)
405 m->saved_queue_if_no_path = m->queue_if_no_path;
406 else
407 m->saved_queue_if_no_path = queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 m->queue_if_no_path = queue_if_no_path;
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -0700409 if (!m->queue_if_no_path && m->queue_size)
Alasdair G Kergonc5573082005-05-05 16:16:07 -0700410 queue_work(kmultipathd, &m->process_queued_ios);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411
412 spin_unlock_irqrestore(&m->lock, flags);
413
414 return 0;
415}
416
417/*-----------------------------------------------------------------
418 * The multipath daemon is responsible for resubmitting queued ios.
419 *---------------------------------------------------------------*/
420
421static void dispatch_queued_ios(struct multipath *m)
422{
423 int r;
424 unsigned long flags;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100425 struct dm_mpath_io *mpio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 union map_info *info;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100427 struct request *clone, *n;
428 LIST_HEAD(cl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429
430 spin_lock_irqsave(&m->lock, flags);
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100431 list_splice_init(&m->queued_ios, &cl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 spin_unlock_irqrestore(&m->lock, flags);
433
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100434 list_for_each_entry_safe(clone, n, &cl, queuelist) {
435 list_del_init(&clone->queuelist);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100437 info = dm_get_rq_mapinfo(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438 mpio = info->ptr;
439
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100440 r = map_io(m, clone, mpio, 1);
441 if (r < 0) {
442 mempool_free(mpio, m->mpio_pool);
443 dm_kill_unmapped_request(clone, r);
444 } else if (r == DM_MAPIO_REMAPPED)
445 dm_dispatch_request(clone);
446 else if (r == DM_MAPIO_REQUEUE) {
447 mempool_free(mpio, m->mpio_pool);
448 dm_requeue_unmapped_request(clone);
449 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 }
451}
452
David Howellsc4028952006-11-22 14:57:56 +0000453static void process_queued_ios(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454{
David Howellsc4028952006-11-22 14:57:56 +0000455 struct multipath *m =
456 container_of(work, struct multipath, process_queued_ios);
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000457 struct pgpath *pgpath = NULL;
Chandra Seetharamane54f77d2009-06-22 10:12:12 +0100458 unsigned must_queue = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 unsigned long flags;
460
461 spin_lock_irqsave(&m->lock, flags);
462
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -0700463 if (!m->queue_size)
464 goto out;
465
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 if (!m->current_pgpath)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +0100467 __choose_pgpath(m, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468
469 pgpath = m->current_pgpath;
470
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -0700471 if ((pgpath && !m->queue_io) ||
472 (!pgpath && !m->queue_if_no_path))
473 must_queue = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
Kiyoshi Uedafb612642010-03-06 02:32:18 +0000475 if (m->pg_init_required && !m->pg_init_in_progress && pgpath)
476 __pg_init_all_paths(m);
477
Alasdair G Kergonc3cd4f62005-07-12 15:53:04 -0700478out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 if (!must_queue)
481 dispatch_queued_ios(m);
482}
483
484/*
485 * An event is triggered whenever a path is taken out of use.
486 * Includes path failure and PG bypass.
487 */
David Howellsc4028952006-11-22 14:57:56 +0000488static void trigger_event(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489{
David Howellsc4028952006-11-22 14:57:56 +0000490 struct multipath *m =
491 container_of(work, struct multipath, trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492
493 dm_table_event(m->ti->table);
494}
495
496/*-----------------------------------------------------------------
497 * Constructor/argument parsing:
498 * <#multipath feature args> [<arg>]*
499 * <#hw_handler args> [hw_handler [<arg>]*]
500 * <#priority groups>
501 * <initial priority group>
502 * [<selector> <#selector args> [<arg>]*
503 * <#paths> <#per-path selector args>
504 * [<path> [<arg>]* ]+ ]+
505 *---------------------------------------------------------------*/
506struct param {
507 unsigned min;
508 unsigned max;
509 char *error;
510};
511
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512static int read_param(struct param *param, char *str, unsigned *v, char **error)
513{
514 if (!str ||
515 (sscanf(str, "%u", v) != 1) ||
516 (*v < param->min) ||
517 (*v > param->max)) {
518 *error = param->error;
519 return -EINVAL;
520 }
521
522 return 0;
523}
524
525struct arg_set {
526 unsigned argc;
527 char **argv;
528};
529
530static char *shift(struct arg_set *as)
531{
532 char *r;
533
534 if (as->argc) {
535 as->argc--;
536 r = *as->argv;
537 as->argv++;
538 return r;
539 }
540
541 return NULL;
542}
543
544static void consume(struct arg_set *as, unsigned n)
545{
546 BUG_ON (as->argc < n);
547 as->argc -= n;
548 as->argv += n;
549}
550
551static int parse_path_selector(struct arg_set *as, struct priority_group *pg,
552 struct dm_target *ti)
553{
554 int r;
555 struct path_selector_type *pst;
556 unsigned ps_argc;
557
558 static struct param _params[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700559 {0, 1024, "invalid number of path selector args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 };
561
562 pst = dm_get_path_selector(shift(as));
563 if (!pst) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700564 ti->error = "unknown path selector type";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 return -EINVAL;
566 }
567
568 r = read_param(_params, shift(as), &ps_argc, &ti->error);
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100569 if (r) {
570 dm_put_path_selector(pst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 return -EINVAL;
Mikulas Patocka371b2e32008-07-21 12:00:24 +0100572 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573
Mikulas Patocka0e0497c2009-06-22 10:08:02 +0100574 if (ps_argc > as->argc) {
575 dm_put_path_selector(pst);
576 ti->error = "not enough arguments for path selector";
577 return -EINVAL;
578 }
579
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 r = pst->create(&pg->ps, ps_argc, as->argv);
581 if (r) {
582 dm_put_path_selector(pst);
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700583 ti->error = "path selector constructor failed";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 return r;
585 }
586
587 pg->ps.type = pst;
588 consume(as, ps_argc);
589
590 return 0;
591}
592
593static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
594 struct dm_target *ti)
595{
596 int r;
597 struct pgpath *p;
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700598 struct multipath *m = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599
600 /* we need at least a path arg */
601 if (as->argc < 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700602 ti->error = "no device given";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100603 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 }
605
606 p = alloc_pgpath();
607 if (!p)
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100608 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
610 r = dm_get_device(ti, shift(as), ti->begin, ti->len,
611 dm_table_get_mode(ti->table), &p->path.dev);
612 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700613 ti->error = "error getting device";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 goto bad;
615 }
616
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700617 if (m->hw_handler_name) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100618 struct request_queue *q = bdev_get_queue(p->path.dev->bdev);
619
620 r = scsi_dh_attach(q, m->hw_handler_name);
621 if (r == -EBUSY) {
622 /*
623 * Already attached to different hw_handler,
624 * try to reattach with correct one.
625 */
626 scsi_dh_detach(q);
627 r = scsi_dh_attach(q, m->hw_handler_name);
628 }
629
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700630 if (r < 0) {
Hannes Reineckea0cf7ea2009-06-22 10:12:11 +0100631 ti->error = "error attaching hardware handler";
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700632 dm_put_device(ti, p->path.dev);
633 goto bad;
634 }
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700635
636 if (m->hw_handler_params) {
637 r = scsi_dh_set_params(q, m->hw_handler_params);
638 if (r < 0) {
639 ti->error = "unable to set hardware "
640 "handler parameters";
641 scsi_dh_detach(q);
642 dm_put_device(ti, p->path.dev);
643 goto bad;
644 }
645 }
Hannes Reineckeae11b1b2008-07-17 17:49:02 -0700646 }
647
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
649 if (r) {
650 dm_put_device(ti, p->path.dev);
651 goto bad;
652 }
653
654 return p;
655
656 bad:
657 free_pgpath(p);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100658 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659}
660
661static struct priority_group *parse_priority_group(struct arg_set *as,
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700662 struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663{
664 static struct param _params[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700665 {1, 1024, "invalid number of paths"},
666 {0, 1024, "invalid number of selector args"}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667 };
668
669 int r;
670 unsigned i, nr_selector_args, nr_params;
671 struct priority_group *pg;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700672 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673
674 if (as->argc < 2) {
675 as->argc = 0;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100676 ti->error = "not enough priority group arguments";
677 return ERR_PTR(-EINVAL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678 }
679
680 pg = alloc_priority_group();
681 if (!pg) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700682 ti->error = "couldn't allocate priority group";
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100683 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 }
685 pg->m = m;
686
687 r = parse_path_selector(as, pg, ti);
688 if (r)
689 goto bad;
690
691 /*
692 * read the paths
693 */
694 r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error);
695 if (r)
696 goto bad;
697
698 r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error);
699 if (r)
700 goto bad;
701
702 nr_params = 1 + nr_selector_args;
703 for (i = 0; i < pg->nr_pgpaths; i++) {
704 struct pgpath *pgpath;
705 struct arg_set path_args;
706
Mikulas Patocka148acff2008-07-21 12:00:30 +0100707 if (as->argc < nr_params) {
708 ti->error = "not enough path parameters";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 goto bad;
Mikulas Patocka148acff2008-07-21 12:00:30 +0100710 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711
712 path_args.argc = nr_params;
713 path_args.argv = as->argv;
714
715 pgpath = parse_path(&path_args, &pg->ps, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100716 if (IS_ERR(pgpath)) {
717 r = PTR_ERR(pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 goto bad;
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100719 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
721 pgpath->pg = pg;
722 list_add_tail(&pgpath->list, &pg->pgpaths);
723 consume(as, nr_params);
724 }
725
726 return pg;
727
728 bad:
729 free_priority_group(pg, ti);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100730 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731}
732
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700733static int parse_hw_handler(struct arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 unsigned hw_argc;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700736 int ret;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700737 struct dm_target *ti = m->ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
739 static struct param _params[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700740 {0, 1024, "invalid number of hardware handler args"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 };
742
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700743 if (read_param(_params, shift(as), &hw_argc, &ti->error))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 return -EINVAL;
745
746 if (!hw_argc)
747 return 0;
748
Mikulas Patockae094f4f2009-06-22 10:12:10 +0100749 if (hw_argc > as->argc) {
750 ti->error = "not enough arguments for hardware handler";
751 return -EINVAL;
752 }
753
Chandra Seetharamancfae5c92008-05-01 14:50:11 -0700754 m->hw_handler_name = kstrdup(shift(as), GFP_KERNEL);
755 request_module("scsi_dh_%s", m->hw_handler_name);
756 if (scsi_dh_handler_exist(m->hw_handler_name) == 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700757 ti->error = "unknown hardware handler type";
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700758 ret = -EINVAL;
759 goto fail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 }
Chandra Seetharaman14e98c52008-11-13 23:39:06 +0000761
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700762 if (hw_argc > 1) {
763 char *p;
764 int i, j, len = 4;
765
766 for (i = 0; i <= hw_argc - 2; i++)
767 len += strlen(as->argv[i]) + 1;
768 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
769 if (!p) {
770 ti->error = "memory allocation failed";
771 ret = -ENOMEM;
772 goto fail;
773 }
774 j = sprintf(p, "%d", hw_argc - 1);
775 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
776 j = sprintf(p, "%s", as->argv[i]);
777 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 consume(as, hw_argc - 1);
779
780 return 0;
Chandra Seetharaman2bfd2e12009-08-03 12:42:45 -0700781fail:
782 kfree(m->hw_handler_name);
783 m->hw_handler_name = NULL;
784 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785}
786
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700787static int parse_features(struct arg_set *as, struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788{
789 int r;
790 unsigned argc;
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700791 struct dm_target *ti = m->ti;
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100792 const char *param_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793
794 static struct param _params[] = {
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100795 {0, 3, "invalid number of feature args"},
796 {1, 50, "pg_init_retries must be between 1 and 50"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 };
798
799 r = read_param(_params, shift(as), &argc, &ti->error);
800 if (r)
801 return -EINVAL;
802
803 if (!argc)
804 return 0;
805
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100806 do {
807 param_name = shift(as);
808 argc--;
809
810 if (!strnicmp(param_name, MESG_STR("queue_if_no_path"))) {
811 r = queue_if_no_path(m, 1, 0);
812 continue;
813 }
814
815 if (!strnicmp(param_name, MESG_STR("pg_init_retries")) &&
816 (argc >= 1)) {
817 r = read_param(_params + 1, shift(as),
818 &m->pg_init_retries, &ti->error);
819 argc--;
820 continue;
821 }
822
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 ti->error = "Unrecognised multipath feature request";
Dave Wysochanskic9e45582007-10-19 22:47:53 +0100824 r = -EINVAL;
825 } while (argc && !r);
826
827 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828}
829
830static int multipath_ctr(struct dm_target *ti, unsigned int argc,
831 char **argv)
832{
833 /* target parameters */
834 static struct param _params[] = {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700835 {1, 1024, "invalid number of priority groups"},
836 {1, 1024, "invalid initial priority group number"},
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 };
838
839 int r;
840 struct multipath *m;
841 struct arg_set as;
842 unsigned pg_count = 0;
843 unsigned next_pg_num;
844
845 as.argc = argc;
846 as.argv = argv;
847
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700848 m = alloc_multipath(ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849 if (!m) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700850 ti->error = "can't allocate multipath";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 return -EINVAL;
852 }
853
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700854 r = parse_features(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 if (r)
856 goto bad;
857
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700858 r = parse_hw_handler(&as, m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 if (r)
860 goto bad;
861
862 r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error);
863 if (r)
864 goto bad;
865
866 r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error);
867 if (r)
868 goto bad;
869
870 /* parse the priority groups */
871 while (as.argc) {
872 struct priority_group *pg;
873
Micha³ Miros³aw28f16c22006-10-03 01:15:33 -0700874 pg = parse_priority_group(&as, m);
Benjamin Marzinski01460f32008-10-10 13:36:57 +0100875 if (IS_ERR(pg)) {
876 r = PTR_ERR(pg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 goto bad;
878 }
879
880 m->nr_valid_paths += pg->nr_pgpaths;
881 list_add_tail(&pg->list, &m->priority_groups);
882 pg_count++;
883 pg->pg_num = pg_count;
884 if (!--next_pg_num)
885 m->next_pg = pg;
886 }
887
888 if (pg_count != m->nr_priority_groups) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700889 ti->error = "priority group count mismatch";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890 r = -EINVAL;
891 goto bad;
892 }
893
Mikulas Patocka86279212009-06-22 10:12:24 +0100894 ti->num_flush_requests = 1;
895
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 return 0;
897
898 bad:
899 free_multipath(m);
900 return r;
901}
902
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000903static void multipath_wait_for_pg_init_completion(struct multipath *m)
904{
905 DECLARE_WAITQUEUE(wait, current);
906 unsigned long flags;
907
908 add_wait_queue(&m->pg_init_wait, &wait);
909
910 while (1) {
911 set_current_state(TASK_UNINTERRUPTIBLE);
912
913 spin_lock_irqsave(&m->lock, flags);
914 if (!m->pg_init_in_progress) {
915 spin_unlock_irqrestore(&m->lock, flags);
916 break;
917 }
918 spin_unlock_irqrestore(&m->lock, flags);
919
920 io_schedule();
921 }
922 set_current_state(TASK_RUNNING);
923
924 remove_wait_queue(&m->pg_init_wait, &wait);
925}
926
927static void flush_multipath_work(struct multipath *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928{
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -0700929 flush_workqueue(kmpath_handlerd);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000930 multipath_wait_for_pg_init_completion(m);
Alasdair G Kergona044d012005-07-12 15:53:02 -0700931 flush_workqueue(kmultipathd);
Mikulas Patocka53b351f2009-06-22 10:12:13 +0100932 flush_scheduled_work();
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +0000933}
934
935static void multipath_dtr(struct dm_target *ti)
936{
937 struct multipath *m = ti->private;
938
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +0000939 flush_multipath_work(m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 free_multipath(m);
941}
942
943/*
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100944 * Map cloned requests
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945 */
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100946static int multipath_map(struct dm_target *ti, struct request *clone,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947 union map_info *map_context)
948{
949 int r;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100950 struct dm_mpath_io *mpio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 struct multipath *m = (struct multipath *) ti->private;
952
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100953 mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
954 if (!mpio)
955 /* ENOMEM, requeue */
956 return DM_MAPIO_REQUEUE;
957 memset(mpio, 0, sizeof(*mpio));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958
959 map_context->ptr = mpio;
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +0100960 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
961 r = map_io(m, clone, mpio, 0);
Kiyoshi Ueda45e15722006-12-08 02:41:10 -0800962 if (r < 0 || r == DM_MAPIO_REQUEUE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 mempool_free(mpio, m->mpio_pool);
964
965 return r;
966}
967
968/*
969 * Take a path out of use.
970 */
971static int fail_path(struct pgpath *pgpath)
972{
973 unsigned long flags;
974 struct multipath *m = pgpath->pg->m;
975
976 spin_lock_irqsave(&m->lock, flags);
977
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100978 if (!pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 goto out;
980
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700981 DMWARN("Failing path %s.", pgpath->path.dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
983 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
Kiyoshi Ueda66800732008-10-10 13:36:58 +0100984 pgpath->is_active = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 pgpath->fail_count++;
986
987 m->nr_valid_paths--;
988
989 if (pgpath == m->current_pgpath)
990 m->current_pgpath = NULL;
991
Mike Andersonb15546f2007-10-19 22:48:02 +0100992 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
993 pgpath->path.dev->name, m->nr_valid_paths);
994
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +0000995 schedule_work(&m->trigger_event);
Mike Anderson224cb3e2008-08-29 09:36:09 +0200996 queue_work(kmultipathd, &pgpath->deactivate_path);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997
998out:
999 spin_unlock_irqrestore(&m->lock, flags);
1000
1001 return 0;
1002}
1003
1004/*
1005 * Reinstate a previously-failed path
1006 */
1007static int reinstate_path(struct pgpath *pgpath)
1008{
1009 int r = 0;
1010 unsigned long flags;
1011 struct multipath *m = pgpath->pg->m;
1012
1013 spin_lock_irqsave(&m->lock, flags);
1014
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001015 if (pgpath->is_active)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 goto out;
1017
Alasdair G Kergondef052d2008-07-21 12:00:31 +01001018 if (!pgpath->pg->ps.type->reinstate_path) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 DMWARN("Reinstate path not supported by path selector %s",
1020 pgpath->pg->ps.type->name);
1021 r = -EINVAL;
1022 goto out;
1023 }
1024
1025 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1026 if (r)
1027 goto out;
1028
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001029 pgpath->is_active = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001031 if (!m->nr_valid_paths++ && m->queue_size) {
1032 m->current_pgpath = NULL;
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001033 queue_work(kmultipathd, &m->process_queued_ios);
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001034 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1035 if (queue_work(kmpath_handlerd, &pgpath->activate_path))
1036 m->pg_init_in_progress++;
1037 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038
Mike Andersonb15546f2007-10-19 22:48:02 +01001039 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1040 pgpath->path.dev->name, m->nr_valid_paths);
1041
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001042 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043
1044out:
1045 spin_unlock_irqrestore(&m->lock, flags);
1046
1047 return r;
1048}
1049
1050/*
1051 * Fail or reinstate all paths that match the provided struct dm_dev.
1052 */
1053static int action_dev(struct multipath *m, struct dm_dev *dev,
1054 action_fn action)
1055{
1056 int r = 0;
1057 struct pgpath *pgpath;
1058 struct priority_group *pg;
1059
1060 list_for_each_entry(pg, &m->priority_groups, list) {
1061 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1062 if (pgpath->path.dev == dev)
1063 r = action(pgpath);
1064 }
1065 }
1066
1067 return r;
1068}
1069
1070/*
1071 * Temporarily try to avoid having to use the specified PG
1072 */
1073static void bypass_pg(struct multipath *m, struct priority_group *pg,
1074 int bypassed)
1075{
1076 unsigned long flags;
1077
1078 spin_lock_irqsave(&m->lock, flags);
1079
1080 pg->bypassed = bypassed;
1081 m->current_pgpath = NULL;
1082 m->current_pg = NULL;
1083
1084 spin_unlock_irqrestore(&m->lock, flags);
1085
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001086 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087}
1088
1089/*
1090 * Switch to using the specified PG from the next I/O that gets mapped
1091 */
1092static int switch_pg_num(struct multipath *m, const char *pgstr)
1093{
1094 struct priority_group *pg;
1095 unsigned pgnum;
1096 unsigned long flags;
1097
1098 if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
1099 (pgnum > m->nr_priority_groups)) {
1100 DMWARN("invalid PG number supplied to switch_pg_num");
1101 return -EINVAL;
1102 }
1103
1104 spin_lock_irqsave(&m->lock, flags);
1105 list_for_each_entry(pg, &m->priority_groups, list) {
1106 pg->bypassed = 0;
1107 if (--pgnum)
1108 continue;
1109
1110 m->current_pgpath = NULL;
1111 m->current_pg = NULL;
1112 m->next_pg = pg;
1113 }
1114 spin_unlock_irqrestore(&m->lock, flags);
1115
Alasdair G Kergonfe9cf302009-01-06 03:05:13 +00001116 schedule_work(&m->trigger_event);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117 return 0;
1118}
1119
1120/*
1121 * Set/clear bypassed status of a PG.
1122 * PGs are numbered upwards from 1 in the order they were declared.
1123 */
1124static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
1125{
1126 struct priority_group *pg;
1127 unsigned pgnum;
1128
1129 if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
1130 (pgnum > m->nr_priority_groups)) {
1131 DMWARN("invalid PG number supplied to bypass_pg");
1132 return -EINVAL;
1133 }
1134
1135 list_for_each_entry(pg, &m->priority_groups, list) {
1136 if (!--pgnum)
1137 break;
1138 }
1139
1140 bypass_pg(m, pg, bypassed);
1141 return 0;
1142}
1143
1144/*
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001145 * Should we retry pg_init immediately?
1146 */
1147static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1148{
1149 unsigned long flags;
1150 int limit_reached = 0;
1151
1152 spin_lock_irqsave(&m->lock, flags);
1153
1154 if (m->pg_init_count <= m->pg_init_retries)
1155 m->pg_init_required = 1;
1156 else
1157 limit_reached = 1;
1158
1159 spin_unlock_irqrestore(&m->lock, flags);
1160
1161 return limit_reached;
1162}
1163
Chandra Seetharaman3ae31f62009-10-21 09:22:46 -07001164static void pg_init_done(void *data, int errors)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001165{
Moger, Babu83c0d5d2010-03-06 02:29:45 +00001166 struct pgpath *pgpath = data;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001167 struct priority_group *pg = pgpath->pg;
1168 struct multipath *m = pg->m;
1169 unsigned long flags;
1170
1171 /* device or driver problems */
1172 switch (errors) {
1173 case SCSI_DH_OK:
1174 break;
1175 case SCSI_DH_NOSYS:
1176 if (!m->hw_handler_name) {
1177 errors = 0;
1178 break;
1179 }
Moger, Babuf7b934c2010-03-06 02:29:49 +00001180 DMERR("Could not failover the device: Handler scsi_dh_%s "
1181 "Error %d.", m->hw_handler_name, errors);
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001182 /*
1183 * Fail path for now, so we do not ping pong
1184 */
1185 fail_path(pgpath);
1186 break;
1187 case SCSI_DH_DEV_TEMP_BUSY:
1188 /*
1189 * Probably doing something like FW upgrade on the
1190 * controller so try the other pg.
1191 */
1192 bypass_pg(m, pg, 1);
1193 break;
1194 /* TODO: For SCSI_DH_RETRY we should wait a couple seconds */
1195 case SCSI_DH_RETRY:
1196 case SCSI_DH_IMM_RETRY:
1197 case SCSI_DH_RES_TEMP_UNAVAIL:
1198 if (pg_init_limit_reached(m, pgpath))
1199 fail_path(pgpath);
1200 errors = 0;
1201 break;
1202 default:
1203 /*
1204 * We probably do not want to fail the path for a device
1205 * error, but this is what the old dm did. In future
1206 * patches we can do more advanced handling.
1207 */
1208 fail_path(pgpath);
1209 }
1210
1211 spin_lock_irqsave(&m->lock, flags);
1212 if (errors) {
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001213 if (pgpath == m->current_pgpath) {
1214 DMERR("Could not failover device. Error %d.", errors);
1215 m->current_pgpath = NULL;
1216 m->current_pg = NULL;
1217 }
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001218 } else if (!m->pg_init_required)
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001219 pg->bypassed = 0;
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001220
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001221 if (--m->pg_init_in_progress)
1222 /* Activations of other paths are still on going */
1223 goto out;
1224
1225 if (!m->pg_init_required)
1226 m->queue_io = 0;
1227
1228 queue_work(kmultipathd, &m->process_queued_ios);
1229
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001230 /*
1231 * Wake up any thread waiting to suspend.
1232 */
1233 wake_up(&m->pg_init_wait);
1234
Kiyoshi Uedad0259bf2010-03-06 02:30:02 +00001235out:
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001236 spin_unlock_irqrestore(&m->lock, flags);
1237}
1238
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001239static void activate_path(struct work_struct *work)
1240{
Chandra Seetharamane54f77d2009-06-22 10:12:12 +01001241 struct pgpath *pgpath =
1242 container_of(work, struct pgpath, activate_path);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001243
Chandra Seetharaman3ae31f62009-10-21 09:22:46 -07001244 scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
Moger, Babu83c0d5d2010-03-06 02:29:45 +00001245 pg_init_done, pgpath);
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001246}
1247
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248/*
1249 * end_io handling
1250 */
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001251static int do_end_io(struct multipath *m, struct request *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001252 int error, struct dm_mpath_io *mpio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253{
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001254 /*
1255 * We don't queue any clone request inside the multipath target
1256 * during end I/O handling, since those clone requests don't have
1257 * bio clones. If we queue them inside the multipath target,
1258 * we need to make bio clones, that requires memory allocation.
1259 * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
1260 * don't have bio clones.)
1261 * Instead of queueing the clone request here, we queue the original
1262 * request into dm core, which will remake a clone request and
1263 * clone bios for it and resubmit it later.
1264 */
1265 int r = DM_ENDIO_REQUEUE;
Stefan Bader640eb3b2005-11-21 21:32:35 -08001266 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001268 if (!error && !clone->errors)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 return 0; /* I/O complete */
1270
Alasdair G Kergonf6a80ea2005-07-12 15:53:01 -07001271 if (error == -EOPNOTSUPP)
1272 return error;
1273
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001274 if (mpio->pgpath)
1275 fail_path(mpio->pgpath);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276
Stefan Bader640eb3b2005-11-21 21:32:35 -08001277 spin_lock_irqsave(&m->lock, flags);
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001278 if (!m->nr_valid_paths && !m->queue_if_no_path && !__must_push_back(m))
1279 r = -EIO;
Stefan Bader640eb3b2005-11-21 21:32:35 -08001280 spin_unlock_irqrestore(&m->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001282 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283}
1284
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001285static int multipath_end_io(struct dm_target *ti, struct request *clone,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 int error, union map_info *map_context)
1287{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001288 struct multipath *m = ti->private;
1289 struct dm_mpath_io *mpio = map_context->ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290 struct pgpath *pgpath = mpio->pgpath;
1291 struct path_selector *ps;
1292 int r;
1293
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001294 r = do_end_io(m, clone, error, mpio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295 if (pgpath) {
1296 ps = &pgpath->pg->ps;
1297 if (ps->type->end_io)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001298 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 }
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001300 mempool_free(mpio, m->mpio_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301
1302 return r;
1303}
1304
1305/*
1306 * Suspend can't complete until all the I/O is processed so if
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001307 * the last path fails we must error any remaining I/O.
1308 * Note that if the freeze_bdev fails while suspending, the
1309 * queue_if_no_path state is lost - userspace should reset it.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 */
1311static void multipath_presuspend(struct dm_target *ti)
1312{
1313 struct multipath *m = (struct multipath *) ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314
Alasdair G Kergon485ef692005-09-27 21:45:45 -07001315 queue_if_no_path(m, 0, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316}
1317
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001318static void multipath_postsuspend(struct dm_target *ti)
1319{
Mike Anderson6380f262009-12-10 23:52:21 +00001320 struct multipath *m = ti->private;
1321
1322 mutex_lock(&m->work_mutex);
Kiyoshi Ueda2bded7b2010-03-06 02:32:13 +00001323 flush_multipath_work(m);
Mike Anderson6380f262009-12-10 23:52:21 +00001324 mutex_unlock(&m->work_mutex);
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001325}
1326
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001327/*
1328 * Restore the queue_if_no_path setting.
1329 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330static void multipath_resume(struct dm_target *ti)
1331{
1332 struct multipath *m = (struct multipath *) ti->private;
1333 unsigned long flags;
1334
1335 spin_lock_irqsave(&m->lock, flags);
Alasdair G Kergon436d4102005-07-12 15:53:03 -07001336 m->queue_if_no_path = m->saved_queue_if_no_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 spin_unlock_irqrestore(&m->lock, flags);
1338}
1339
1340/*
1341 * Info output has the following format:
1342 * num_multipath_feature_args [multipath_feature_args]*
1343 * num_handler_status_args [handler_status_args]*
1344 * num_groups init_group_number
1345 * [A|D|E num_ps_status_args [ps_status_args]*
1346 * num_paths num_selector_args
1347 * [path_dev A|F fail_count [selector_args]* ]+ ]+
1348 *
1349 * Table output has the following format (identical to the constructor string):
1350 * num_feature_args [features_args]*
1351 * num_handler_args hw_handler [hw_handler_args]*
1352 * num_groups init_group_number
1353 * [priority selector-name num_ps_args [ps_args]*
1354 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1355 */
1356static int multipath_status(struct dm_target *ti, status_type_t type,
1357 char *result, unsigned int maxlen)
1358{
1359 int sz = 0;
1360 unsigned long flags;
1361 struct multipath *m = (struct multipath *) ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362 struct priority_group *pg;
1363 struct pgpath *p;
1364 unsigned pg_num;
1365 char state;
1366
1367 spin_lock_irqsave(&m->lock, flags);
1368
1369 /* Features */
1370 if (type == STATUSTYPE_INFO)
Dave Wysochanskic9e45582007-10-19 22:47:53 +01001371 DMEMIT("2 %u %u ", m->queue_size, m->pg_init_count);
1372 else {
1373 DMEMIT("%u ", m->queue_if_no_path +
1374 (m->pg_init_retries > 0) * 2);
1375 if (m->queue_if_no_path)
1376 DMEMIT("queue_if_no_path ");
1377 if (m->pg_init_retries)
1378 DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1379 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001381 if (!m->hw_handler_name || type == STATUSTYPE_INFO)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 DMEMIT("0 ");
1383 else
Chandra Seetharamancfae5c92008-05-01 14:50:11 -07001384 DMEMIT("1 %s ", m->hw_handler_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001385
1386 DMEMIT("%u ", m->nr_priority_groups);
1387
1388 if (m->next_pg)
1389 pg_num = m->next_pg->pg_num;
1390 else if (m->current_pg)
1391 pg_num = m->current_pg->pg_num;
1392 else
1393 pg_num = 1;
1394
1395 DMEMIT("%u ", pg_num);
1396
1397 switch (type) {
1398 case STATUSTYPE_INFO:
1399 list_for_each_entry(pg, &m->priority_groups, list) {
1400 if (pg->bypassed)
1401 state = 'D'; /* Disabled */
1402 else if (pg == m->current_pg)
1403 state = 'A'; /* Currently Active */
1404 else
1405 state = 'E'; /* Enabled */
1406
1407 DMEMIT("%c ", state);
1408
1409 if (pg->ps.type->status)
1410 sz += pg->ps.type->status(&pg->ps, NULL, type,
1411 result + sz,
1412 maxlen - sz);
1413 else
1414 DMEMIT("0 ");
1415
1416 DMEMIT("%u %u ", pg->nr_pgpaths,
1417 pg->ps.type->info_args);
1418
1419 list_for_each_entry(p, &pg->pgpaths, list) {
1420 DMEMIT("%s %s %u ", p->path.dev->name,
Kiyoshi Ueda66800732008-10-10 13:36:58 +01001421 p->is_active ? "A" : "F",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422 p->fail_count);
1423 if (pg->ps.type->status)
1424 sz += pg->ps.type->status(&pg->ps,
1425 &p->path, type, result + sz,
1426 maxlen - sz);
1427 }
1428 }
1429 break;
1430
1431 case STATUSTYPE_TABLE:
1432 list_for_each_entry(pg, &m->priority_groups, list) {
1433 DMEMIT("%s ", pg->ps.type->name);
1434
1435 if (pg->ps.type->status)
1436 sz += pg->ps.type->status(&pg->ps, NULL, type,
1437 result + sz,
1438 maxlen - sz);
1439 else
1440 DMEMIT("0 ");
1441
1442 DMEMIT("%u %u ", pg->nr_pgpaths,
1443 pg->ps.type->table_args);
1444
1445 list_for_each_entry(p, &pg->pgpaths, list) {
1446 DMEMIT("%s ", p->path.dev->name);
1447 if (pg->ps.type->status)
1448 sz += pg->ps.type->status(&pg->ps,
1449 &p->path, type, result + sz,
1450 maxlen - sz);
1451 }
1452 }
1453 break;
1454 }
1455
1456 spin_unlock_irqrestore(&m->lock, flags);
1457
1458 return 0;
1459}
1460
1461static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1462{
Mike Anderson6380f262009-12-10 23:52:21 +00001463 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464 struct dm_dev *dev;
1465 struct multipath *m = (struct multipath *) ti->private;
1466 action_fn action;
1467
Mike Anderson6380f262009-12-10 23:52:21 +00001468 mutex_lock(&m->work_mutex);
1469
Kiyoshi Uedac2f3d242009-12-10 23:52:27 +00001470 if (dm_suspended(ti)) {
1471 r = -EBUSY;
1472 goto out;
1473 }
1474
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475 if (argc == 1) {
Mike Anderson6380f262009-12-10 23:52:21 +00001476 if (!strnicmp(argv[0], MESG_STR("queue_if_no_path"))) {
1477 r = queue_if_no_path(m, 1, 0);
1478 goto out;
1479 } else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path"))) {
1480 r = queue_if_no_path(m, 0, 0);
1481 goto out;
1482 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483 }
1484
Mike Anderson6380f262009-12-10 23:52:21 +00001485 if (argc != 2) {
1486 DMWARN("Unrecognised multipath message received.");
1487 goto out;
1488 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489
Mike Anderson6380f262009-12-10 23:52:21 +00001490 if (!strnicmp(argv[0], MESG_STR("disable_group"))) {
1491 r = bypass_pg_num(m, argv[1], 1);
1492 goto out;
1493 } else if (!strnicmp(argv[0], MESG_STR("enable_group"))) {
1494 r = bypass_pg_num(m, argv[1], 0);
1495 goto out;
1496 } else if (!strnicmp(argv[0], MESG_STR("switch_group"))) {
1497 r = switch_pg_num(m, argv[1]);
1498 goto out;
1499 } else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 action = reinstate_path;
1501 else if (!strnicmp(argv[0], MESG_STR("fail_path")))
1502 action = fail_path;
Mike Anderson6380f262009-12-10 23:52:21 +00001503 else {
1504 DMWARN("Unrecognised multipath message received.");
1505 goto out;
1506 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001507
1508 r = dm_get_device(ti, argv[1], ti->begin, ti->len,
1509 dm_table_get_mode(ti->table), &dev);
1510 if (r) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001511 DMWARN("message: error getting device %s",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512 argv[1]);
Mike Anderson6380f262009-12-10 23:52:21 +00001513 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 }
1515
1516 r = action_dev(m, dev, action);
1517
1518 dm_put_device(ti, dev);
1519
Mike Anderson6380f262009-12-10 23:52:21 +00001520out:
1521 mutex_unlock(&m->work_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523}
1524
Al Viro647b3d02007-08-28 22:15:59 -04001525static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
Milan Broz9af4aa32006-10-03 01:15:20 -07001526 unsigned long arg)
1527{
1528 struct multipath *m = (struct multipath *) ti->private;
1529 struct block_device *bdev = NULL;
Al Viro633a08b2007-08-29 20:34:12 -04001530 fmode_t mode = 0;
Milan Broz9af4aa32006-10-03 01:15:20 -07001531 unsigned long flags;
1532 int r = 0;
1533
1534 spin_lock_irqsave(&m->lock, flags);
1535
1536 if (!m->current_pgpath)
Kiyoshi Ueda02ab8232009-06-22 10:12:27 +01001537 __choose_pgpath(m, 0);
Milan Broz9af4aa32006-10-03 01:15:20 -07001538
Milan Broze90dae12006-10-03 01:15:22 -07001539 if (m->current_pgpath) {
Milan Broz9af4aa32006-10-03 01:15:20 -07001540 bdev = m->current_pgpath->path.dev->bdev;
Al Viro633a08b2007-08-29 20:34:12 -04001541 mode = m->current_pgpath->path.dev->mode;
Milan Broze90dae12006-10-03 01:15:22 -07001542 }
Milan Broz9af4aa32006-10-03 01:15:20 -07001543
1544 if (m->queue_io)
1545 r = -EAGAIN;
1546 else if (!bdev)
1547 r = -EIO;
1548
1549 spin_unlock_irqrestore(&m->lock, flags);
1550
Al Viro633a08b2007-08-29 20:34:12 -04001551 return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
Milan Broz9af4aa32006-10-03 01:15:20 -07001552}
1553
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001554static int multipath_iterate_devices(struct dm_target *ti,
1555 iterate_devices_callout_fn fn, void *data)
1556{
1557 struct multipath *m = ti->private;
1558 struct priority_group *pg;
1559 struct pgpath *p;
1560 int ret = 0;
1561
1562 list_for_each_entry(pg, &m->priority_groups, list) {
1563 list_for_each_entry(p, &pg->pgpaths, list) {
Mike Snitzer5dea2712009-07-23 20:30:42 +01001564 ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001565 if (ret)
1566 goto out;
1567 }
1568 }
1569
1570out:
1571 return ret;
1572}
1573
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001574static int __pgpath_busy(struct pgpath *pgpath)
1575{
1576 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1577
1578 return dm_underlying_device_busy(q);
1579}
1580
1581/*
1582 * We return "busy", only when we can map I/Os but underlying devices
1583 * are busy (so even if we map I/Os now, the I/Os will wait on
1584 * the underlying queue).
1585 * In other words, if we want to kill I/Os or queue them inside us
1586 * due to map unavailability, we don't return "busy". Otherwise,
1587 * dm core won't give us the I/Os and we can't do what we want.
1588 */
1589static int multipath_busy(struct dm_target *ti)
1590{
1591 int busy = 0, has_active = 0;
1592 struct multipath *m = ti->private;
1593 struct priority_group *pg;
1594 struct pgpath *pgpath;
1595 unsigned long flags;
1596
1597 spin_lock_irqsave(&m->lock, flags);
1598
1599 /* Guess which priority_group will be used at next mapping time */
1600 if (unlikely(!m->current_pgpath && m->next_pg))
1601 pg = m->next_pg;
1602 else if (likely(m->current_pg))
1603 pg = m->current_pg;
1604 else
1605 /*
1606 * We don't know which pg will be used at next mapping time.
1607 * We don't call __choose_pgpath() here to avoid to trigger
1608 * pg_init just by busy checking.
1609 * So we don't know whether underlying devices we will be using
1610 * at next mapping time are busy or not. Just try mapping.
1611 */
1612 goto out;
1613
1614 /*
1615 * If there is one non-busy active path at least, the path selector
1616 * will be able to select it. So we consider such a pg as not busy.
1617 */
1618 busy = 1;
1619 list_for_each_entry(pgpath, &pg->pgpaths, list)
1620 if (pgpath->is_active) {
1621 has_active = 1;
1622
1623 if (!__pgpath_busy(pgpath)) {
1624 busy = 0;
1625 break;
1626 }
1627 }
1628
1629 if (!has_active)
1630 /*
1631 * No active path in this pg, so this pg won't be used and
1632 * the current_pg will be changed at next mapping time.
1633 * We need to try mapping to determine it.
1634 */
1635 busy = 0;
1636
1637out:
1638 spin_unlock_irqrestore(&m->lock, flags);
1639
1640 return busy;
1641}
1642
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643/*-----------------------------------------------------------------
1644 * Module setup
1645 *---------------------------------------------------------------*/
1646static struct target_type multipath_target = {
1647 .name = "multipath",
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001648 .version = {1, 1, 1},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649 .module = THIS_MODULE,
1650 .ctr = multipath_ctr,
1651 .dtr = multipath_dtr,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001652 .map_rq = multipath_map,
1653 .rq_end_io = multipath_end_io,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 .presuspend = multipath_presuspend,
Kiyoshi Ueda6df400a2009-12-10 23:52:19 +00001655 .postsuspend = multipath_postsuspend,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001656 .resume = multipath_resume,
1657 .status = multipath_status,
1658 .message = multipath_message,
Milan Broz9af4aa32006-10-03 01:15:20 -07001659 .ioctl = multipath_ioctl,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001660 .iterate_devices = multipath_iterate_devices,
Kiyoshi Uedaf40c67f2009-06-22 10:12:37 +01001661 .busy = multipath_busy,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662};
1663
1664static int __init dm_multipath_init(void)
1665{
1666 int r;
1667
1668 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001669 _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670 if (!_mpio_cache)
1671 return -ENOMEM;
1672
1673 r = dm_register_target(&multipath_target);
1674 if (r < 0) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001675 DMERR("register failed %d", r);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 kmem_cache_destroy(_mpio_cache);
1677 return -EINVAL;
1678 }
1679
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001680 kmultipathd = create_workqueue("kmpathd");
1681 if (!kmultipathd) {
Alasdair G Kergon0cd33122007-07-12 17:27:01 +01001682 DMERR("failed to create workqueue kmpathd");
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001683 dm_unregister_target(&multipath_target);
1684 kmem_cache_destroy(_mpio_cache);
1685 return -ENOMEM;
1686 }
1687
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001688 /*
1689 * A separate workqueue is used to handle the device handlers
1690 * to avoid overloading existing workqueue. Overloading the
1691 * old workqueue would also create a bottleneck in the
1692 * path of the storage hardware device activation.
1693 */
1694 kmpath_handlerd = create_singlethread_workqueue("kmpath_handlerd");
1695 if (!kmpath_handlerd) {
1696 DMERR("failed to create workqueue kmpath_handlerd");
1697 destroy_workqueue(kmultipathd);
1698 dm_unregister_target(&multipath_target);
1699 kmem_cache_destroy(_mpio_cache);
1700 return -ENOMEM;
1701 }
1702
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001703 DMINFO("version %u.%u.%u loaded",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 multipath_target.version[0], multipath_target.version[1],
1705 multipath_target.version[2]);
1706
1707 return r;
1708}
1709
1710static void __exit dm_multipath_exit(void)
1711{
Chandra Seetharamanbab7cfc2008-05-01 14:50:22 -07001712 destroy_workqueue(kmpath_handlerd);
Alasdair G Kergonc5573082005-05-05 16:16:07 -07001713 destroy_workqueue(kmultipathd);
1714
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001715 dm_unregister_target(&multipath_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716 kmem_cache_destroy(_mpio_cache);
1717}
1718
Linus Torvalds1da177e2005-04-16 15:20:36 -07001719module_init(dm_multipath_init);
1720module_exit(dm_multipath_exit);
1721
1722MODULE_DESCRIPTION(DM_NAME " multipath target");
1723MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
1724MODULE_LICENSE("GPL");