Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Functions related to barrier IO handling |
| 3 | */ |
| 4 | #include <linux/kernel.h> |
| 5 | #include <linux/module.h> |
| 6 | #include <linux/bio.h> |
| 7 | #include <linux/blkdev.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 8 | #include <linux/gfp.h> |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 9 | |
| 10 | #include "blk.h" |
| 11 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 12 | static struct request *queue_next_fseq(struct request_queue *q); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 13 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 14 | unsigned blk_flush_cur_seq(struct request_queue *q) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 15 | { |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 16 | if (!q->flush_seq) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 17 | return 0; |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 18 | return 1 << ffz(q->flush_seq); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 19 | } |
| 20 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 21 | static struct request *blk_flush_complete_seq(struct request_queue *q, |
| 22 | unsigned seq, int error) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 23 | { |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 24 | struct request *next_rq = NULL; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 25 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 26 | if (error && !q->flush_err) |
| 27 | q->flush_err = error; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 28 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 29 | BUG_ON(q->flush_seq & seq); |
| 30 | q->flush_seq |= seq; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 31 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 32 | if (blk_flush_cur_seq(q) != QUEUE_FSEQ_DONE) { |
| 33 | /* not complete yet, queue the next flush sequence */ |
| 34 | next_rq = queue_next_fseq(q); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 35 | } else { |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 36 | /* complete this flush request */ |
| 37 | __blk_end_request_all(q->orig_flush_rq, q->flush_err); |
| 38 | q->orig_flush_rq = NULL; |
| 39 | q->flush_seq = 0; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 40 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 41 | /* dispatch the next flush if there's one */ |
| 42 | if (!list_empty(&q->pending_flushes)) { |
| 43 | next_rq = list_entry_rq(q->pending_flushes.next); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 44 | list_move(&next_rq->queuelist, &q->queue_head); |
| 45 | } |
| 46 | } |
| 47 | return next_rq; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 48 | } |
| 49 | |
| 50 | static void pre_flush_end_io(struct request *rq, int error) |
| 51 | { |
| 52 | elv_completed_request(rq->q, rq); |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 53 | blk_flush_complete_seq(rq->q, QUEUE_FSEQ_PREFLUSH, error); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 54 | } |
| 55 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 56 | static void flush_data_end_io(struct request *rq, int error) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 57 | { |
| 58 | elv_completed_request(rq->q, rq); |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 59 | blk_flush_complete_seq(rq->q, QUEUE_FSEQ_DATA, error); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 60 | } |
| 61 | |
| 62 | static void post_flush_end_io(struct request *rq, int error) |
| 63 | { |
| 64 | elv_completed_request(rq->q, rq); |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 65 | blk_flush_complete_seq(rq->q, QUEUE_FSEQ_POSTFLUSH, error); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 66 | } |
| 67 | |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 68 | static void queue_flush(struct request_queue *q, struct request *rq, |
| 69 | rq_end_io_fn *end_io) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 70 | { |
FUJITA Tomonori | 2a4aa30 | 2008-04-29 09:54:36 +0200 | [diff] [blame] | 71 | blk_rq_init(q, rq); |
FUJITA Tomonori | 28e18d0 | 2010-07-09 09:38:24 +0900 | [diff] [blame] | 72 | rq->cmd_type = REQ_TYPE_FS; |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 73 | rq->cmd_flags = REQ_FLUSH; |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 74 | rq->rq_disk = q->orig_flush_rq->rq_disk; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 75 | rq->end_io = end_io; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 76 | |
| 77 | elv_insert(q, rq, ELEVATOR_INSERT_FRONT); |
| 78 | } |
| 79 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 80 | static struct request *queue_next_fseq(struct request_queue *q) |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 81 | { |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 82 | struct request *rq = &q->flush_rq; |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 83 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 84 | switch (blk_flush_cur_seq(q)) { |
| 85 | case QUEUE_FSEQ_PREFLUSH: |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 86 | queue_flush(q, rq, pre_flush_end_io); |
| 87 | break; |
| 88 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 89 | case QUEUE_FSEQ_DATA: |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 90 | /* initialize proxy request and queue it */ |
| 91 | blk_rq_init(q, rq); |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 92 | init_request_from_bio(rq, q->orig_flush_rq->bio); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 93 | rq->cmd_flags &= ~REQ_HARDBARRIER; |
| 94 | if (q->ordered & QUEUE_ORDERED_DO_FUA) |
| 95 | rq->cmd_flags |= REQ_FUA; |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 96 | rq->end_io = flush_data_end_io; |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 97 | |
| 98 | elv_insert(q, rq, ELEVATOR_INSERT_FRONT); |
| 99 | break; |
| 100 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 101 | case QUEUE_FSEQ_POSTFLUSH: |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 102 | queue_flush(q, rq, post_flush_end_io); |
| 103 | break; |
| 104 | |
| 105 | default: |
| 106 | BUG(); |
| 107 | } |
| 108 | return rq; |
| 109 | } |
| 110 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 111 | struct request *blk_do_flush(struct request_queue *q, struct request *rq) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 112 | { |
Tejun Heo | 8f11b3e | 2008-11-28 13:32:05 +0900 | [diff] [blame] | 113 | unsigned skip = 0; |
| 114 | |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 115 | if (!(rq->cmd_flags & REQ_HARDBARRIER)) |
| 116 | return rq; |
| 117 | |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 118 | if (q->flush_seq) { |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 119 | /* |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 120 | * Sequenced flush is already in progress and they |
| 121 | * can't be processed in parallel. Queue for later |
| 122 | * processing. |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 123 | */ |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 124 | list_move_tail(&rq->queuelist, &q->pending_flushes); |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 125 | return NULL; |
| 126 | } |
| 127 | |
| 128 | if (unlikely(q->next_ordered == QUEUE_ORDERED_NONE)) { |
| 129 | /* |
| 130 | * Queue ordering not supported. Terminate |
| 131 | * with prejudice. |
| 132 | */ |
| 133 | blk_dequeue_request(rq); |
| 134 | __blk_end_request_all(rq, -EOPNOTSUPP); |
| 135 | return NULL; |
| 136 | } |
| 137 | |
| 138 | /* |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 139 | * Start a new flush sequence |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 140 | */ |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 141 | q->flush_err = 0; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 142 | q->ordered = q->next_ordered; |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 143 | q->flush_seq |= QUEUE_FSEQ_STARTED; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 144 | |
Tejun Heo | 58eea92 | 2008-11-28 13:32:06 +0900 | [diff] [blame] | 145 | /* |
| 146 | * For an empty barrier, there's no actual BAR request, which |
| 147 | * in turn makes POSTFLUSH unnecessary. Mask them off. |
| 148 | */ |
Tejun Heo | 6958f14 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 149 | if (!blk_rq_sectors(rq)) |
Tejun Heo | 58eea92 | 2008-11-28 13:32:06 +0900 | [diff] [blame] | 150 | q->ordered &= ~(QUEUE_ORDERED_DO_BAR | |
| 151 | QUEUE_ORDERED_DO_POSTFLUSH); |
| 152 | |
Tejun Heo | f671620 | 2008-11-28 13:32:04 +0900 | [diff] [blame] | 153 | /* stash away the original request */ |
Tejun Heo | 9934c8c | 2009-05-08 11:54:16 +0900 | [diff] [blame] | 154 | blk_dequeue_request(rq); |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 155 | q->orig_flush_rq = rq; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 156 | |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 157 | if (!(q->ordered & QUEUE_ORDERED_DO_PREFLUSH)) |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 158 | skip |= QUEUE_FSEQ_PREFLUSH; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 159 | |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 160 | if (!(q->ordered & QUEUE_ORDERED_DO_BAR)) |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 161 | skip |= QUEUE_FSEQ_DATA; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 162 | |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 163 | if (!(q->ordered & QUEUE_ORDERED_DO_POSTFLUSH)) |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 164 | skip |= QUEUE_FSEQ_POSTFLUSH; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 165 | |
Tejun Heo | 28e7d18 | 2010-09-03 11:56:16 +0200 | [diff] [blame] | 166 | /* complete skipped sequences and return the first sequence */ |
Tejun Heo | dd4c133 | 2010-09-03 11:56:16 +0200 | [diff] [blame^] | 167 | return blk_flush_complete_seq(q, skip, 0); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | static void bio_end_empty_barrier(struct bio *bio, int err) |
| 171 | { |
Jens Axboe | cc66b45 | 2008-03-04 11:47:46 +0100 | [diff] [blame] | 172 | if (err) { |
| 173 | if (err == -EOPNOTSUPP) |
| 174 | set_bit(BIO_EOPNOTSUPP, &bio->bi_flags); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 175 | clear_bit(BIO_UPTODATE, &bio->bi_flags); |
Jens Axboe | cc66b45 | 2008-03-04 11:47:46 +0100 | [diff] [blame] | 176 | } |
Dmitry Monakhov | f17e232 | 2010-04-28 17:55:07 +0400 | [diff] [blame] | 177 | if (bio->bi_private) |
| 178 | complete(bio->bi_private); |
| 179 | bio_put(bio); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 180 | } |
| 181 | |
| 182 | /** |
| 183 | * blkdev_issue_flush - queue a flush |
| 184 | * @bdev: blockdev to issue flush for |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 185 | * @gfp_mask: memory allocation flags (for bio_alloc) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 186 | * @error_sector: error sector |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 187 | * @flags: BLKDEV_IFL_* flags to control behaviour |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 188 | * |
| 189 | * Description: |
| 190 | * Issue a flush for the block device in question. Caller can supply |
| 191 | * room for storing the error offset in case of a flush error, if they |
Dmitry Monakhov | f17e232 | 2010-04-28 17:55:07 +0400 | [diff] [blame] | 192 | * wish to. If WAIT flag is not passed then caller may check only what |
| 193 | * request was pushed in some internal queue for later handling. |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 194 | */ |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 195 | int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, |
| 196 | sector_t *error_sector, unsigned long flags) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 197 | { |
| 198 | DECLARE_COMPLETION_ONSTACK(wait); |
| 199 | struct request_queue *q; |
| 200 | struct bio *bio; |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 201 | int ret = 0; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 202 | |
| 203 | if (bdev->bd_disk == NULL) |
| 204 | return -ENXIO; |
| 205 | |
| 206 | q = bdev_get_queue(bdev); |
| 207 | if (!q) |
| 208 | return -ENXIO; |
| 209 | |
Dave Chinner | f10d9f6 | 2010-07-13 17:50:50 +1000 | [diff] [blame] | 210 | /* |
| 211 | * some block devices may not have their queue correctly set up here |
| 212 | * (e.g. loop device without a backing file) and so issuing a flush |
| 213 | * here will panic. Ensure there is a request function before issuing |
| 214 | * the barrier. |
| 215 | */ |
| 216 | if (!q->make_request_fn) |
| 217 | return -ENXIO; |
| 218 | |
Dmitry Monakhov | fbd9b09 | 2010-04-28 17:55:06 +0400 | [diff] [blame] | 219 | bio = bio_alloc(gfp_mask, 0); |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 220 | bio->bi_end_io = bio_end_empty_barrier; |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 221 | bio->bi_bdev = bdev; |
Dmitry Monakhov | f17e232 | 2010-04-28 17:55:07 +0400 | [diff] [blame] | 222 | if (test_bit(BLKDEV_WAIT, &flags)) |
| 223 | bio->bi_private = &wait; |
| 224 | |
| 225 | bio_get(bio); |
OGAWA Hirofumi | 2ebca85 | 2008-08-11 17:07:08 +0100 | [diff] [blame] | 226 | submit_bio(WRITE_BARRIER, bio); |
Dmitry Monakhov | f17e232 | 2010-04-28 17:55:07 +0400 | [diff] [blame] | 227 | if (test_bit(BLKDEV_WAIT, &flags)) { |
| 228 | wait_for_completion(&wait); |
| 229 | /* |
| 230 | * The driver must store the error location in ->bi_sector, if |
| 231 | * it supports it. For non-stacked drivers, this should be |
| 232 | * copied from blk_rq_pos(rq). |
| 233 | */ |
| 234 | if (error_sector) |
| 235 | *error_sector = bio->bi_sector; |
| 236 | } |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 237 | |
Jens Axboe | cc66b45 | 2008-03-04 11:47:46 +0100 | [diff] [blame] | 238 | if (bio_flagged(bio, BIO_EOPNOTSUPP)) |
| 239 | ret = -EOPNOTSUPP; |
| 240 | else if (!bio_flagged(bio, BIO_UPTODATE)) |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 241 | ret = -EIO; |
| 242 | |
| 243 | bio_put(bio); |
| 244 | return ret; |
| 245 | } |
Jens Axboe | 86db1e2 | 2008-01-29 14:53:40 +0100 | [diff] [blame] | 246 | EXPORT_SYMBOL(blkdev_issue_flush); |