blob: 46fe9248410d74c0c457695a81b3febdc31c0a41 [file] [log] [blame]
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +04001/*
2 * Functions related to generic helpers functions
3 */
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/bio.h>
7#include <linux/blkdev.h>
8#include <linux/scatterlist.h>
9
10#include "blk.h"
11
Mike Christie4e49ea42016-06-05 14:31:41 -050012static struct bio *next_bio(struct bio *bio, unsigned int nr_pages,
Christoph Hellwig9082e872016-04-16 14:55:27 -040013 gfp_t gfp)
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040014{
Christoph Hellwig9082e872016-04-16 14:55:27 -040015 struct bio *new = bio_alloc(gfp, nr_pages);
Lukas Czerner5dba3082011-05-06 19:26:27 -060016
Christoph Hellwig9082e872016-04-16 14:55:27 -040017 if (bio) {
18 bio_chain(bio, new);
Mike Christie4e49ea42016-06-05 14:31:41 -050019 submit_bio(bio);
Christoph Hellwig9082e872016-04-16 14:55:27 -040020 }
21
22 return new;
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040023}
24
Christoph Hellwig38f25252016-04-16 14:55:28 -040025int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
Christoph Hellwig288dab82016-06-09 16:00:36 +020026 sector_t nr_sects, gfp_t gfp_mask, int flags,
Mike Christie469e3212016-06-05 14:31:49 -050027 struct bio **biop)
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040028{
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040029 struct request_queue *q = bdev_get_queue(bdev);
Christoph Hellwig38f25252016-04-16 14:55:28 -040030 struct bio *bio = *biop;
Ming Lina22c4d72015-10-22 09:59:42 -070031 unsigned int granularity;
Christoph Hellwig288dab82016-06-09 16:00:36 +020032 enum req_op op;
Ming Lina22c4d72015-10-22 09:59:42 -070033 int alignment;
Darrick J. Wong28b2be22016-10-11 13:51:08 -070034 sector_t bs_mask;
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040035
36 if (!q)
37 return -ENXIO;
Christoph Hellwig288dab82016-06-09 16:00:36 +020038
39 if (flags & BLKDEV_DISCARD_SECURE) {
Christoph Hellwige950fdf2016-07-19 11:23:33 +020040 if (flags & BLKDEV_DISCARD_ZERO)
41 return -EOPNOTSUPP;
Christoph Hellwig288dab82016-06-09 16:00:36 +020042 if (!blk_queue_secure_erase(q))
43 return -EOPNOTSUPP;
44 op = REQ_OP_SECURE_ERASE;
45 } else {
46 if (!blk_queue_discard(q))
47 return -EOPNOTSUPP;
Christoph Hellwige950fdf2016-07-19 11:23:33 +020048 if ((flags & BLKDEV_DISCARD_ZERO) &&
49 !q->limits.discard_zeroes_data)
50 return -EOPNOTSUPP;
Christoph Hellwig288dab82016-06-09 16:00:36 +020051 op = REQ_OP_DISCARD;
52 }
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040053
Darrick J. Wong28b2be22016-10-11 13:51:08 -070054 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
55 if ((sector | nr_sects) & bs_mask)
56 return -EINVAL;
57
Ming Lina22c4d72015-10-22 09:59:42 -070058 /* Zero-sector (unknown) and one-sector granularities are the same. */
59 granularity = max(q->limits.discard_granularity >> 9, 1U);
60 alignment = (bdev_discard_alignment(bdev) >> 9) % granularity;
61
Lukas Czerner5dba3082011-05-06 19:26:27 -060062 while (nr_sects) {
Paolo Bonzinic6e66632012-08-02 09:48:50 +020063 unsigned int req_sects;
Ming Lina22c4d72015-10-22 09:59:42 -070064 sector_t end_sect, tmp;
Paolo Bonzinic6e66632012-08-02 09:48:50 +020065
Ming Lina22c4d72015-10-22 09:59:42 -070066 /* Make sure bi_size doesn't overflow */
67 req_sects = min_t(sector_t, nr_sects, UINT_MAX >> 9);
68
Christoph Hellwig9082e872016-04-16 14:55:27 -040069 /**
Ming Lina22c4d72015-10-22 09:59:42 -070070 * If splitting a request, and the next starting sector would be
71 * misaligned, stop the discard at the previous aligned sector.
72 */
Paolo Bonzinic6e66632012-08-02 09:48:50 +020073 end_sect = sector + req_sects;
Ming Lina22c4d72015-10-22 09:59:42 -070074 tmp = end_sect;
75 if (req_sects < nr_sects &&
76 sector_div(tmp, granularity) != alignment) {
77 end_sect = end_sect - alignment;
78 sector_div(end_sect, granularity);
79 end_sect = end_sect * granularity + alignment;
80 req_sects = end_sect - sector;
81 }
Paolo Bonzinic6e66632012-08-02 09:48:50 +020082
Mike Christie4e49ea42016-06-05 14:31:41 -050083 bio = next_bio(bio, 1, gfp_mask);
Kent Overstreet4f024f32013-10-11 15:44:27 -070084 bio->bi_iter.bi_sector = sector;
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040085 bio->bi_bdev = bdev;
Christoph Hellwig288dab82016-06-09 16:00:36 +020086 bio_set_op_attrs(bio, op, 0);
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040087
Kent Overstreet4f024f32013-10-11 15:44:27 -070088 bio->bi_iter.bi_size = req_sects << 9;
Paolo Bonzinic6e66632012-08-02 09:48:50 +020089 nr_sects -= req_sects;
90 sector = end_sect;
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +040091
Jens Axboec8123f82014-02-12 09:34:01 -070092 /*
93 * We can loop for a long time in here, if someone does
94 * full device discards (like mkfs). Be nice and allow
95 * us to schedule out to avoid softlocking if preempt
96 * is disabled.
97 */
98 cond_resched();
Lukas Czerner5dba3082011-05-06 19:26:27 -060099 }
Christoph Hellwig38f25252016-04-16 14:55:28 -0400100
101 *biop = bio;
102 return 0;
103}
104EXPORT_SYMBOL(__blkdev_issue_discard);
105
106/**
107 * blkdev_issue_discard - queue a discard
108 * @bdev: blockdev to issue discard for
109 * @sector: start sector
110 * @nr_sects: number of sectors to discard
111 * @gfp_mask: memory allocation flags (for bio_alloc)
112 * @flags: BLKDEV_IFL_* flags to control behaviour
113 *
114 * Description:
115 * Issue a discard request for the sectors in question.
116 */
117int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
118 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
119{
Christoph Hellwig38f25252016-04-16 14:55:28 -0400120 struct bio *bio = NULL;
121 struct blk_plug plug;
122 int ret;
123
Christoph Hellwig38f25252016-04-16 14:55:28 -0400124 blk_start_plug(&plug);
Christoph Hellwig288dab82016-06-09 16:00:36 +0200125 ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags,
Christoph Hellwig38f25252016-04-16 14:55:28 -0400126 &bio);
Mike Snitzerbbd848e0f2016-05-05 11:54:21 -0400127 if (!ret && bio) {
Mike Christie4e49ea42016-06-05 14:31:41 -0500128 ret = submit_bio_wait(bio);
Christoph Hellwige950fdf2016-07-19 11:23:33 +0200129 if (ret == -EOPNOTSUPP && !(flags & BLKDEV_DISCARD_ZERO))
Mike Snitzerbbd848e0f2016-05-05 11:54:21 -0400130 ret = 0;
Shaun Tancheff05bd92d2016-06-07 11:32:13 -0500131 bio_put(bio);
Mike Snitzerbbd848e0f2016-05-05 11:54:21 -0400132 }
Shaohua Li0cfbcaf2012-12-14 11:15:51 +0800133 blk_finish_plug(&plug);
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +0400134
Mike Snitzerbbd848e0f2016-05-05 11:54:21 -0400135 return ret;
Dmitry Monakhovf31e7e42010-04-28 17:55:08 +0400136}
137EXPORT_SYMBOL(blkdev_issue_discard);
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400138
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400139/**
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400140 * blkdev_issue_write_same - queue a write same operation
141 * @bdev: target blockdev
142 * @sector: start sector
143 * @nr_sects: number of sectors to write
144 * @gfp_mask: memory allocation flags (for bio_alloc)
145 * @page: page containing data to write
146 *
147 * Description:
148 * Issue a write same request for the sectors in question.
149 */
150int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
151 sector_t nr_sects, gfp_t gfp_mask,
152 struct page *page)
153{
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400154 struct request_queue *q = bdev_get_queue(bdev);
155 unsigned int max_write_same_sectors;
Christoph Hellwig9082e872016-04-16 14:55:27 -0400156 struct bio *bio = NULL;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400157 int ret = 0;
Darrick J. Wong28b2be22016-10-11 13:51:08 -0700158 sector_t bs_mask;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400159
160 if (!q)
161 return -ENXIO;
162
Darrick J. Wong28b2be22016-10-11 13:51:08 -0700163 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
164 if ((sector | nr_sects) & bs_mask)
165 return -EINVAL;
166
Ming Linb49a0872015-05-22 00:46:56 -0700167 /* Ensure that max_write_same_sectors doesn't overflow bi_size */
168 max_write_same_sectors = UINT_MAX >> 9;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400169
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400170 while (nr_sects) {
Mike Christie4e49ea42016-06-05 14:31:41 -0500171 bio = next_bio(bio, 1, gfp_mask);
Kent Overstreet4f024f32013-10-11 15:44:27 -0700172 bio->bi_iter.bi_sector = sector;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400173 bio->bi_bdev = bdev;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400174 bio->bi_vcnt = 1;
175 bio->bi_io_vec->bv_page = page;
176 bio->bi_io_vec->bv_offset = 0;
177 bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);
Mike Christie95fe6c12016-06-05 14:31:48 -0500178 bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0);
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400179
180 if (nr_sects > max_write_same_sectors) {
Kent Overstreet4f024f32013-10-11 15:44:27 -0700181 bio->bi_iter.bi_size = max_write_same_sectors << 9;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400182 nr_sects -= max_write_same_sectors;
183 sector += max_write_same_sectors;
184 } else {
Kent Overstreet4f024f32013-10-11 15:44:27 -0700185 bio->bi_iter.bi_size = nr_sects << 9;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400186 nr_sects = 0;
187 }
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400188 }
189
Shaun Tancheff05bd92d2016-06-07 11:32:13 -0500190 if (bio) {
Mike Christie4e49ea42016-06-05 14:31:41 -0500191 ret = submit_bio_wait(bio);
Shaun Tancheff05bd92d2016-06-07 11:32:13 -0500192 bio_put(bio);
193 }
Christoph Hellwig3f40bf22016-07-19 11:23:34 +0200194 return ret;
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400195}
196EXPORT_SYMBOL(blkdev_issue_write_same);
197
198/**
Ben Hutchings291d24f2011-03-01 13:45:24 -0500199 * blkdev_issue_zeroout - generate number of zero filed write bios
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400200 * @bdev: blockdev to issue
201 * @sector: start sector
202 * @nr_sects: number of sectors to write
203 * @gfp_mask: memory allocation flags (for bio_alloc)
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400204 *
205 * Description:
206 * Generate and issue number of bios with zerofiled pages.
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400207 */
208
Fabian Frederick35086782014-05-26 22:19:14 +0200209static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
210 sector_t nr_sects, gfp_t gfp_mask)
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400211{
Dmitry Monakhov18edc8e2010-08-06 13:23:25 +0200212 int ret;
Christoph Hellwig9082e872016-04-16 14:55:27 -0400213 struct bio *bio = NULL;
Lukas Czerner0aeea182011-03-11 10:23:53 +0100214 unsigned int sz;
Darrick J. Wong28b2be22016-10-11 13:51:08 -0700215 sector_t bs_mask;
216
217 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
218 if ((sector | nr_sects) & bs_mask)
219 return -EINVAL;
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400220
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400221 while (nr_sects != 0) {
Mike Christie4e49ea42016-06-05 14:31:41 -0500222 bio = next_bio(bio, min(nr_sects, (sector_t)BIO_MAX_PAGES),
Christoph Hellwig9082e872016-04-16 14:55:27 -0400223 gfp_mask);
Kent Overstreet4f024f32013-10-11 15:44:27 -0700224 bio->bi_iter.bi_sector = sector;
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400225 bio->bi_bdev = bdev;
Mike Christie95fe6c12016-06-05 14:31:48 -0500226 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400227
Jens Axboe0341aaf2010-04-29 09:28:21 +0200228 while (nr_sects != 0) {
229 sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400230 ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
231 nr_sects -= ret >> 9;
232 sector += ret >> 9;
233 if (ret < (sz << 9))
234 break;
235 }
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400236 }
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400237
Shaun Tancheff05bd92d2016-06-07 11:32:13 -0500238 if (bio) {
Linus Torvaldsd05d7f42016-07-26 15:03:07 -0700239 ret = submit_bio_wait(bio);
Shaun Tancheff05bd92d2016-06-07 11:32:13 -0500240 bio_put(bio);
241 return ret;
242 }
Christoph Hellwig9082e872016-04-16 14:55:27 -0400243 return 0;
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400244}
Martin K. Petersen579e8f32012-09-18 12:19:28 -0400245
246/**
247 * blkdev_issue_zeroout - zero-fill a block range
248 * @bdev: blockdev to write
249 * @sector: start sector
250 * @nr_sects: number of sectors to write
251 * @gfp_mask: memory allocation flags (for bio_alloc)
Martin K. Petersend93ba7a2015-01-20 20:06:30 -0500252 * @discard: whether to discard the block range
Martin K. Petersen579e8f32012-09-18 12:19:28 -0400253 *
254 * Description:
Martin K. Petersend93ba7a2015-01-20 20:06:30 -0500255 * Zero-fill a block range. If the discard flag is set and the block
256 * device guarantees that subsequent READ operations to the block range
257 * in question will return zeroes, the blocks will be discarded. Should
258 * the discard request fail, if the discard flag is not set, or if
259 * discard_zeroes_data is not supported, this function will resort to
260 * zeroing the blocks manually, thus provisioning (allocating,
261 * anchoring) them. If the block device supports the WRITE SAME command
262 * blkdev_issue_zeroout() will use it to optimize the process of
263 * clearing the block range. Otherwise the zeroing will be performed
264 * using regular WRITE calls.
Martin K. Petersen579e8f32012-09-18 12:19:28 -0400265 */
266
267int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
Martin K. Petersend93ba7a2015-01-20 20:06:30 -0500268 sector_t nr_sects, gfp_t gfp_mask, bool discard)
Martin K. Petersen579e8f32012-09-18 12:19:28 -0400269{
Christoph Hellwige950fdf2016-07-19 11:23:33 +0200270 if (discard) {
271 if (!blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask,
272 BLKDEV_DISCARD_ZERO))
273 return 0;
274 }
Martin K. Petersend93ba7a2015-01-20 20:06:30 -0500275
Martin K. Petersen9f9ee1f2015-02-05 10:14:54 -0700276 if (bdev_write_same(bdev) &&
277 blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
278 ZERO_PAGE(0)) == 0)
279 return 0;
Martin K. Petersen579e8f32012-09-18 12:19:28 -0400280
281 return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask);
282}
Dmitry Monakhov3f14d792010-04-28 17:55:09 +0400283EXPORT_SYMBOL(blkdev_issue_zeroout);