blob: 27fd8d92892d47e3478d9bcf3e1294a8eb28da89 [file] [log] [blame]
Jens Axboe86db1e22008-01-29 14:53:40 +01001/*
2 * Functions related to mapping data to requests
3 */
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/bio.h>
7#include <linux/blkdev.h>
Kent Overstreet26e49cf2015-01-18 16:16:31 +01008#include <linux/uio.h>
Jens Axboe86db1e22008-01-29 14:53:40 +01009
10#include "blk.h"
11
Christoph Hellwig98d61d52016-07-19 11:31:51 +020012/*
13 * Append a bio to a passthrough request. Only works can be merged into
14 * the request based on the driver constraints.
15 */
16int blk_rq_append_bio(struct request *rq, struct bio *bio)
Jens Axboe86db1e22008-01-29 14:53:40 +010017{
Christoph Hellwig98d61d52016-07-19 11:31:51 +020018 if (!rq->bio) {
19 blk_rq_bio_prep(rq->q, rq, bio);
20 } else {
21 if (!ll_back_merge_fn(rq->q, rq, bio))
22 return -EINVAL;
23
Jens Axboe86db1e22008-01-29 14:53:40 +010024 rq->biotail->bi_next = bio;
25 rq->biotail = bio;
Kent Overstreet4f024f32013-10-11 15:44:27 -070026 rq->__data_len += bio->bi_iter.bi_size;
Jens Axboe86db1e22008-01-29 14:53:40 +010027 }
Christoph Hellwig98d61d52016-07-19 11:31:51 +020028
Jens Axboe86db1e22008-01-29 14:53:40 +010029 return 0;
30}
Christoph Hellwig98d61d52016-07-19 11:31:51 +020031EXPORT_SYMBOL(blk_rq_append_bio);
Jens Axboe86db1e22008-01-29 14:53:40 +010032
33static int __blk_rq_unmap_user(struct bio *bio)
34{
35 int ret = 0;
36
37 if (bio) {
38 if (bio_flagged(bio, BIO_USER_MAPPED))
39 bio_unmap_user(bio);
40 else
41 ret = bio_uncopy_user(bio);
42 }
43
44 return ret;
45}
46
Christoph Hellwig4d6af732016-03-02 18:07:14 +010047static int __blk_rq_map_user_iov(struct request *rq,
48 struct rq_map_data *map_data, struct iov_iter *iter,
49 gfp_t gfp_mask, bool copy)
50{
51 struct request_queue *q = rq->q;
52 struct bio *bio, *orig_bio;
53 int ret;
54
55 if (copy)
56 bio = bio_copy_user_iov(q, map_data, iter, gfp_mask);
57 else
58 bio = bio_map_user_iov(q, iter, gfp_mask);
59
60 if (IS_ERR(bio))
61 return PTR_ERR(bio);
62
63 if (map_data && map_data->null_mapped)
64 bio_set_flag(bio, BIO_NULL_MAPPED);
65
66 iov_iter_advance(iter, bio->bi_iter.bi_size);
67 if (map_data)
68 map_data->offset += bio->bi_iter.bi_size;
69
70 orig_bio = bio;
71 blk_queue_bounce(q, &bio);
72
73 /*
74 * We link the bounce buffer in and could have to traverse it
75 * later so we have to get a ref to prevent it from being freed
76 */
77 bio_get(bio);
78
Christoph Hellwig98d61d52016-07-19 11:31:51 +020079 ret = blk_rq_append_bio(rq, bio);
Christoph Hellwig4d6af732016-03-02 18:07:14 +010080 if (ret) {
81 bio_endio(bio);
82 __blk_rq_unmap_user(orig_bio);
83 bio_put(bio);
84 return ret;
85 }
86
87 return 0;
88}
89
Jens Axboe86db1e22008-01-29 14:53:40 +010090/**
Randy Dunlap710027a2008-08-19 20:13:11 +020091 * blk_rq_map_user_iov - map user data to a request, for REQ_TYPE_BLOCK_PC usage
Jens Axboe86db1e22008-01-29 14:53:40 +010092 * @q: request queue where request should be inserted
93 * @rq: request to map data to
FUJITA Tomonori152e2832008-08-28 16:17:06 +090094 * @map_data: pointer to the rq_map_data holding pages (if necessary)
Kent Overstreet26e49cf2015-01-18 16:16:31 +010095 * @iter: iovec iterator
FUJITA Tomonoria3bce902008-08-28 16:17:05 +090096 * @gfp_mask: memory allocation flags
Jens Axboe86db1e22008-01-29 14:53:40 +010097 *
98 * Description:
Randy Dunlap710027a2008-08-19 20:13:11 +020099 * Data will be mapped directly for zero copy I/O, if possible. Otherwise
Jens Axboe86db1e22008-01-29 14:53:40 +0100100 * a kernel bounce buffer is used.
101 *
Randy Dunlap710027a2008-08-19 20:13:11 +0200102 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while
Jens Axboe86db1e22008-01-29 14:53:40 +0100103 * still in process context.
104 *
105 * Note: The mapped bio may need to be bounced through blk_queue_bounce()
106 * before being submitted to the device, as pages mapped may be out of
107 * reach. It's the callers responsibility to make sure this happens. The
108 * original bio must be passed back in to blk_rq_unmap_user() for proper
109 * unmapping.
110 */
111int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
Kent Overstreet26e49cf2015-01-18 16:16:31 +0100112 struct rq_map_data *map_data,
113 const struct iov_iter *iter, gfp_t gfp_mask)
Jens Axboe86db1e22008-01-29 14:53:40 +0100114{
Al Viro357f4352016-04-08 19:05:19 -0400115 bool copy = false;
116 unsigned long align = q->dma_pad_mask | queue_dma_alignment(q);
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100117 struct bio *bio = NULL;
118 struct iov_iter i;
119 int ret;
Jens Axboe86db1e22008-01-29 14:53:40 +0100120
Linus Torvaldsa0ac4022016-12-06 16:18:14 -0800121 if (!iter_is_iovec(iter))
122 goto fail;
123
Al Viro357f4352016-04-08 19:05:19 -0400124 if (map_data)
125 copy = true;
126 else if (iov_iter_alignment(iter) & align)
127 copy = true;
128 else if (queue_virt_boundary(q))
129 copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter);
FUJITA Tomonoriafdc1a72008-04-11 12:56:51 +0200130
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100131 i = *iter;
132 do {
133 ret =__blk_rq_map_user_iov(rq, map_data, &i, gfp_mask, copy);
134 if (ret)
135 goto unmap_rq;
136 if (!bio)
137 bio = rq->bio;
138 } while (iov_iter_count(&i));
Jens Axboe86db1e22008-01-29 14:53:40 +0100139
FUJITA Tomonorif18573a2008-04-11 12:56:52 +0200140 if (!bio_flagged(bio, BIO_USER_MAPPED))
141 rq->cmd_flags |= REQ_COPY_USER;
Jens Axboe86db1e22008-01-29 14:53:40 +0100142 return 0;
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100143
144unmap_rq:
145 __blk_rq_unmap_user(bio);
Linus Torvaldsa0ac4022016-12-06 16:18:14 -0800146fail:
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100147 rq->bio = NULL;
148 return -EINVAL;
Jens Axboe86db1e22008-01-29 14:53:40 +0100149}
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900150EXPORT_SYMBOL(blk_rq_map_user_iov);
Jens Axboe86db1e22008-01-29 14:53:40 +0100151
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100152int blk_rq_map_user(struct request_queue *q, struct request *rq,
153 struct rq_map_data *map_data, void __user *ubuf,
154 unsigned long len, gfp_t gfp_mask)
155{
Kent Overstreet26e49cf2015-01-18 16:16:31 +0100156 struct iovec iov;
157 struct iov_iter i;
Al Viro8f7e8852015-03-21 20:06:04 -0400158 int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i);
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100159
Al Viro8f7e8852015-03-21 20:06:04 -0400160 if (unlikely(ret < 0))
161 return ret;
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100162
Kent Overstreet26e49cf2015-01-18 16:16:31 +0100163 return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask);
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100164}
165EXPORT_SYMBOL(blk_rq_map_user);
166
Jens Axboe86db1e22008-01-29 14:53:40 +0100167/**
168 * blk_rq_unmap_user - unmap a request with user data
169 * @bio: start of bio list
170 *
171 * Description:
172 * Unmap a rq previously mapped by blk_rq_map_user(). The caller must
173 * supply the original rq->bio from the blk_rq_map_user() return, since
Randy Dunlap710027a2008-08-19 20:13:11 +0200174 * the I/O completion may have changed rq->bio.
Jens Axboe86db1e22008-01-29 14:53:40 +0100175 */
176int blk_rq_unmap_user(struct bio *bio)
177{
178 struct bio *mapped_bio;
179 int ret = 0, ret2;
180
181 while (bio) {
182 mapped_bio = bio;
183 if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
184 mapped_bio = bio->bi_private;
185
186 ret2 = __blk_rq_unmap_user(mapped_bio);
187 if (ret2 && !ret)
188 ret = ret2;
189
190 mapped_bio = bio;
191 bio = bio->bi_next;
192 bio_put(mapped_bio);
193 }
194
195 return ret;
196}
Jens Axboe86db1e22008-01-29 14:53:40 +0100197EXPORT_SYMBOL(blk_rq_unmap_user);
198
199/**
Randy Dunlap710027a2008-08-19 20:13:11 +0200200 * blk_rq_map_kern - map kernel data to a request, for REQ_TYPE_BLOCK_PC usage
Jens Axboe86db1e22008-01-29 14:53:40 +0100201 * @q: request queue where request should be inserted
202 * @rq: request to fill
203 * @kbuf: the kernel buffer
204 * @len: length of user data
205 * @gfp_mask: memory allocation flags
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200206 *
207 * Description:
208 * Data will be mapped directly if possible. Otherwise a bounce
Masanari Iidae2278672014-02-18 22:54:36 +0900209 * buffer is used. Can be called multiple times to append multiple
James Bottomley3a5a3922009-05-17 18:55:18 +0300210 * buffers.
Jens Axboe86db1e22008-01-29 14:53:40 +0100211 */
212int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
213 unsigned int len, gfp_t gfp_mask)
214{
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200215 int reading = rq_data_dir(rq) == READ;
Namhyung Kim14417792010-09-15 13:08:27 +0200216 unsigned long addr = (unsigned long) kbuf;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200217 int do_copy = 0;
Jens Axboe86db1e22008-01-29 14:53:40 +0100218 struct bio *bio;
James Bottomley3a5a3922009-05-17 18:55:18 +0300219 int ret;
Jens Axboe86db1e22008-01-29 14:53:40 +0100220
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400221 if (len > (queue_max_hw_sectors(q) << 9))
Jens Axboe86db1e22008-01-29 14:53:40 +0100222 return -EINVAL;
223 if (!len || !kbuf)
224 return -EINVAL;
225
Namhyung Kim14417792010-09-15 13:08:27 +0200226 do_copy = !blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf);
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200227 if (do_copy)
228 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);
229 else
230 bio = bio_map_kern(q, kbuf, len, gfp_mask);
231
Jens Axboe86db1e22008-01-29 14:53:40 +0100232 if (IS_ERR(bio))
233 return PTR_ERR(bio);
234
majianpeng609f6ea2011-12-21 15:27:24 +0100235 if (!reading)
Mike Christie95fe6c12016-06-05 14:31:48 -0500236 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
Jens Axboe86db1e22008-01-29 14:53:40 +0100237
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200238 if (do_copy)
239 rq->cmd_flags |= REQ_COPY_USER;
240
Christoph Hellwig98d61d52016-07-19 11:31:51 +0200241 ret = blk_rq_append_bio(rq, bio);
James Bottomley3a5a3922009-05-17 18:55:18 +0300242 if (unlikely(ret)) {
243 /* request is too big */
244 bio_put(bio);
245 return ret;
246 }
247
Jens Axboe86db1e22008-01-29 14:53:40 +0100248 blk_queue_bounce(q, &rq->bio);
Jens Axboe86db1e22008-01-29 14:53:40 +0100249 return 0;
250}
Jens Axboe86db1e22008-01-29 14:53:40 +0100251EXPORT_SYMBOL(blk_rq_map_kern);