| /* |
| * Functions to sequence FLUSH and FUA writes. |
| */ |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/bio.h> |
| #include <linux/blkdev.h> |
| #include <linux/gfp.h> |
| |
| #include "blk.h" |
| |
| /* FLUSH/FUA sequences */ |
| enum { |
| QUEUE_FSEQ_STARTED = (1 << 0), /* flushing in progress */ |
| QUEUE_FSEQ_PREFLUSH = (1 << 1), /* pre-flushing in progress */ |
| QUEUE_FSEQ_DATA = (1 << 2), /* data write in progress */ |
| QUEUE_FSEQ_POSTFLUSH = (1 << 3), /* post-flushing in progress */ |
| QUEUE_FSEQ_DONE = (1 << 4), |
| }; |
| |
| static struct request *queue_next_fseq(struct request_queue *q); |
| |
| unsigned blk_flush_cur_seq(struct request_queue *q) |
| { |
| if (!q->flush_seq) |
| return 0; |
| return 1 << ffz(q->flush_seq); |
| } |
| |
| static struct request *blk_flush_complete_seq(struct request_queue *q, |
| unsigned seq, int error) |
| { |
| struct request *next_rq = NULL; |
| |
| if (error && !q->flush_err) |
| q->flush_err = error; |
| |
| BUG_ON(q->flush_seq & seq); |
| q->flush_seq |= seq; |
| |
| if (blk_flush_cur_seq(q) != QUEUE_FSEQ_DONE) { |
| /* not complete yet, queue the next flush sequence */ |
| next_rq = queue_next_fseq(q); |
| } else { |
| /* complete this flush request */ |
| __blk_end_request_all(q->orig_flush_rq, q->flush_err); |
| q->orig_flush_rq = NULL; |
| q->flush_seq = 0; |
| |
| /* dispatch the next flush if there's one */ |
| if (!list_empty(&q->pending_flushes)) { |
| next_rq = list_entry_rq(q->pending_flushes.next); |
| list_move(&next_rq->queuelist, &q->queue_head); |
| } |
| } |
| return next_rq; |
| } |
| |
| static void blk_flush_complete_seq_end_io(struct request_queue *q, |
| unsigned seq, int error) |
| { |
| bool was_empty = elv_queue_empty(q); |
| struct request *next_rq; |
| |
| next_rq = blk_flush_complete_seq(q, seq, error); |
| |
| /* |
| * Moving a request silently to empty queue_head may stall the |
| * queue. Kick the queue in those cases. |
| */ |
| if (was_empty && next_rq) |
| __blk_run_queue(q); |
| } |
| |
| static void pre_flush_end_io(struct request *rq, int error) |
| { |
| elv_completed_request(rq->q, rq); |
| blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_PREFLUSH, error); |
| } |
| |
| static void flush_data_end_io(struct request *rq, int error) |
| { |
| elv_completed_request(rq->q, rq); |
| blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_DATA, error); |
| } |
| |
| static void post_flush_end_io(struct request *rq, int error) |
| { |
| elv_completed_request(rq->q, rq); |
| blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_POSTFLUSH, error); |
| } |
| |
| static void init_flush_request(struct request *rq, struct gendisk *disk) |
| { |
| rq->cmd_type = REQ_TYPE_FS; |
| rq->cmd_flags = WRITE_FLUSH; |
| rq->rq_disk = disk; |
| } |
| |
| static struct request *queue_next_fseq(struct request_queue *q) |
| { |
| struct request *orig_rq = q->orig_flush_rq; |
| struct request *rq = &q->flush_rq; |
| |
| blk_rq_init(q, rq); |
| |
| switch (blk_flush_cur_seq(q)) { |
| case QUEUE_FSEQ_PREFLUSH: |
| init_flush_request(rq, orig_rq->rq_disk); |
| rq->end_io = pre_flush_end_io; |
| break; |
| case QUEUE_FSEQ_DATA: |
| init_request_from_bio(rq, orig_rq->bio); |
| /* |
| * orig_rq->rq_disk may be different from |
| * bio->bi_bdev->bd_disk if orig_rq got here through |
| * remapping drivers. Make sure rq->rq_disk points |
| * to the same one as orig_rq. |
| */ |
| rq->rq_disk = orig_rq->rq_disk; |
| rq->cmd_flags &= ~(REQ_FLUSH | REQ_FUA); |
| rq->cmd_flags |= orig_rq->cmd_flags & (REQ_FLUSH | REQ_FUA); |
| rq->end_io = flush_data_end_io; |
| break; |
| case QUEUE_FSEQ_POSTFLUSH: |
| init_flush_request(rq, orig_rq->rq_disk); |
| rq->end_io = post_flush_end_io; |
| break; |
| default: |
| BUG(); |
| } |
| |
| rq->cmd_flags |= REQ_FLUSH_SEQ; |
| elv_insert(q, rq, ELEVATOR_INSERT_FRONT); |
| return rq; |
| } |
| |
| struct request *blk_do_flush(struct request_queue *q, struct request *rq) |
| { |
| unsigned int fflags = q->flush_flags; /* may change, cache it */ |
| bool has_flush = fflags & REQ_FLUSH, has_fua = fflags & REQ_FUA; |
| bool do_preflush = has_flush && (rq->cmd_flags & REQ_FLUSH); |
| bool do_postflush = has_flush && !has_fua && (rq->cmd_flags & REQ_FUA); |
| unsigned skip = 0; |
| |
| /* |
| * Special case. If there's data but flush is not necessary, |
| * the request can be issued directly. |
| * |
| * Flush w/o data should be able to be issued directly too but |
| * currently some drivers assume that rq->bio contains |
| * non-zero data if it isn't NULL and empty FLUSH requests |
| * getting here usually have bio's without data. |
| */ |
| if (blk_rq_sectors(rq) && !do_preflush && !do_postflush) { |
| rq->cmd_flags &= ~REQ_FLUSH; |
| if (!has_fua) |
| rq->cmd_flags &= ~REQ_FUA; |
| return rq; |
| } |
| |
| /* |
| * Sequenced flushes can't be processed in parallel. If |
| * another one is already in progress, queue for later |
| * processing. |
| */ |
| if (q->flush_seq) { |
| list_move_tail(&rq->queuelist, &q->pending_flushes); |
| return NULL; |
| } |
| |
| /* |
| * Start a new flush sequence |
| */ |
| q->flush_err = 0; |
| q->flush_seq |= QUEUE_FSEQ_STARTED; |
| |
| /* adjust FLUSH/FUA of the original request and stash it away */ |
| rq->cmd_flags &= ~REQ_FLUSH; |
| if (!has_fua) |
| rq->cmd_flags &= ~REQ_FUA; |
| blk_dequeue_request(rq); |
| q->orig_flush_rq = rq; |
| |
| /* skip unneded sequences and return the first one */ |
| if (!do_preflush) |
| skip |= QUEUE_FSEQ_PREFLUSH; |
| if (!blk_rq_sectors(rq)) |
| skip |= QUEUE_FSEQ_DATA; |
| if (!do_postflush) |
| skip |= QUEUE_FSEQ_POSTFLUSH; |
| return blk_flush_complete_seq(q, skip, 0); |
| } |
| |
| static void bio_end_flush(struct bio *bio, int err) |
| { |
| if (err) |
| clear_bit(BIO_UPTODATE, &bio->bi_flags); |
| if (bio->bi_private) |
| complete(bio->bi_private); |
| bio_put(bio); |
| } |
| |
| /** |
| * blkdev_issue_flush - queue a flush |
| * @bdev: blockdev to issue flush for |
| * @gfp_mask: memory allocation flags (for bio_alloc) |
| * @error_sector: error sector |
| * |
| * Description: |
| * Issue a flush for the block device in question. Caller can supply |
| * room for storing the error offset in case of a flush error, if they |
| * wish to. If WAIT flag is not passed then caller may check only what |
| * request was pushed in some internal queue for later handling. |
| */ |
| int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, |
| sector_t *error_sector) |
| { |
| DECLARE_COMPLETION_ONSTACK(wait); |
| struct request_queue *q; |
| struct bio *bio; |
| int ret = 0; |
| |
| if (bdev->bd_disk == NULL) |
| return -ENXIO; |
| |
| q = bdev_get_queue(bdev); |
| if (!q) |
| return -ENXIO; |
| |
| /* |
| * some block devices may not have their queue correctly set up here |
| * (e.g. loop device without a backing file) and so issuing a flush |
| * here will panic. Ensure there is a request function before issuing |
| * the flush. |
| */ |
| if (!q->make_request_fn) |
| return -ENXIO; |
| |
| bio = bio_alloc(gfp_mask, 0); |
| bio->bi_end_io = bio_end_flush; |
| bio->bi_bdev = bdev; |
| bio->bi_private = &wait; |
| |
| bio_get(bio); |
| submit_bio(WRITE_FLUSH, bio); |
| wait_for_completion(&wait); |
| |
| /* |
| * The driver must store the error location in ->bi_sector, if |
| * it supports it. For non-stacked drivers, this should be |
| * copied from blk_rq_pos(rq). |
| */ |
| if (error_sector) |
| *error_sector = bio->bi_sector; |
| |
| if (!bio_flagged(bio, BIO_UPTODATE)) |
| ret = -EIO; |
| |
| bio_put(bio); |
| return ret; |
| } |
| EXPORT_SYMBOL(blkdev_issue_flush); |