blob: 48b8c27acabb9b0c817418f55b8958ae355e8a9b [file] [log] [blame]
Al Viro4f18cd32014-02-05 19:11:33 -05001#include <linux/export.h>
2#include <linux/uio.h>
3#include <linux/pagemap.h>
Al Viro91f79c42014-03-21 04:58:33 -04004#include <linux/slab.h>
5#include <linux/vmalloc.h>
Al Viro241699c2016-09-22 16:33:12 -04006#include <linux/splice.h>
Al Viroa604ec72014-11-24 01:08:00 -05007#include <net/checksum.h>
Al Viro4f18cd32014-02-05 19:11:33 -05008
Al Viro241699c2016-09-22 16:33:12 -04009#define PIPE_PARANOIA /* for now */
10
Al Viro04a31162014-11-27 13:51:41 -050011#define iterate_iovec(i, n, __v, __p, skip, STEP) { \
12 size_t left; \
13 size_t wanted = n; \
14 __p = i->iov; \
15 __v.iov_len = min(n, __p->iov_len - skip); \
16 if (likely(__v.iov_len)) { \
17 __v.iov_base = __p->iov_base + skip; \
18 left = (STEP); \
19 __v.iov_len -= left; \
20 skip += __v.iov_len; \
21 n -= __v.iov_len; \
22 } else { \
23 left = 0; \
24 } \
25 while (unlikely(!left && n)) { \
26 __p++; \
27 __v.iov_len = min(n, __p->iov_len); \
28 if (unlikely(!__v.iov_len)) \
29 continue; \
30 __v.iov_base = __p->iov_base; \
31 left = (STEP); \
32 __v.iov_len -= left; \
33 skip = __v.iov_len; \
34 n -= __v.iov_len; \
35 } \
36 n = wanted - n; \
37}
38
Al Viroa2804552014-11-27 14:48:42 -050039#define iterate_kvec(i, n, __v, __p, skip, STEP) { \
40 size_t wanted = n; \
41 __p = i->kvec; \
42 __v.iov_len = min(n, __p->iov_len - skip); \
43 if (likely(__v.iov_len)) { \
44 __v.iov_base = __p->iov_base + skip; \
45 (void)(STEP); \
46 skip += __v.iov_len; \
47 n -= __v.iov_len; \
48 } \
49 while (unlikely(n)) { \
50 __p++; \
51 __v.iov_len = min(n, __p->iov_len); \
52 if (unlikely(!__v.iov_len)) \
53 continue; \
54 __v.iov_base = __p->iov_base; \
55 (void)(STEP); \
56 skip = __v.iov_len; \
57 n -= __v.iov_len; \
58 } \
59 n = wanted; \
60}
61
Ming Lei1bdc76a2016-05-30 21:34:32 +080062#define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
63 struct bvec_iter __start; \
64 __start.bi_size = n; \
65 __start.bi_bvec_done = skip; \
66 __start.bi_idx = 0; \
67 for_each_bvec(__v, i->bvec, __bi, __start) { \
68 if (!__v.bv_len) \
Al Viro04a31162014-11-27 13:51:41 -050069 continue; \
Al Viro04a31162014-11-27 13:51:41 -050070 (void)(STEP); \
Al Viro04a31162014-11-27 13:51:41 -050071 } \
Al Viro04a31162014-11-27 13:51:41 -050072}
73
Al Viroa2804552014-11-27 14:48:42 -050074#define iterate_all_kinds(i, n, v, I, B, K) { \
Al Viro04a31162014-11-27 13:51:41 -050075 size_t skip = i->iov_offset; \
76 if (unlikely(i->type & ITER_BVEC)) { \
Al Viro04a31162014-11-27 13:51:41 -050077 struct bio_vec v; \
Ming Lei1bdc76a2016-05-30 21:34:32 +080078 struct bvec_iter __bi; \
79 iterate_bvec(i, n, v, __bi, skip, (B)) \
Al Viroa2804552014-11-27 14:48:42 -050080 } else if (unlikely(i->type & ITER_KVEC)) { \
81 const struct kvec *kvec; \
82 struct kvec v; \
83 iterate_kvec(i, n, v, kvec, skip, (K)) \
Al Viro04a31162014-11-27 13:51:41 -050084 } else { \
85 const struct iovec *iov; \
86 struct iovec v; \
87 iterate_iovec(i, n, v, iov, skip, (I)) \
88 } \
89}
90
Al Viroa2804552014-11-27 14:48:42 -050091#define iterate_and_advance(i, n, v, I, B, K) { \
Al Virodd254f52016-05-09 11:54:48 -040092 if (unlikely(i->count < n)) \
93 n = i->count; \
Al Viro19f18452016-05-25 17:36:19 -040094 if (i->count) { \
Al Virodd254f52016-05-09 11:54:48 -040095 size_t skip = i->iov_offset; \
96 if (unlikely(i->type & ITER_BVEC)) { \
Ming Lei1bdc76a2016-05-30 21:34:32 +080097 const struct bio_vec *bvec = i->bvec; \
Al Virodd254f52016-05-09 11:54:48 -040098 struct bio_vec v; \
Ming Lei1bdc76a2016-05-30 21:34:32 +080099 struct bvec_iter __bi; \
100 iterate_bvec(i, n, v, __bi, skip, (B)) \
101 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
102 i->nr_segs -= i->bvec - bvec; \
103 skip = __bi.bi_bvec_done; \
Al Virodd254f52016-05-09 11:54:48 -0400104 } else if (unlikely(i->type & ITER_KVEC)) { \
105 const struct kvec *kvec; \
106 struct kvec v; \
107 iterate_kvec(i, n, v, kvec, skip, (K)) \
108 if (skip == kvec->iov_len) { \
109 kvec++; \
110 skip = 0; \
111 } \
112 i->nr_segs -= kvec - i->kvec; \
113 i->kvec = kvec; \
114 } else { \
115 const struct iovec *iov; \
116 struct iovec v; \
117 iterate_iovec(i, n, v, iov, skip, (I)) \
118 if (skip == iov->iov_len) { \
119 iov++; \
120 skip = 0; \
121 } \
122 i->nr_segs -= iov - i->iov; \
123 i->iov = iov; \
Al Viro7ce2a912014-11-27 13:59:45 -0500124 } \
Al Virodd254f52016-05-09 11:54:48 -0400125 i->count -= n; \
126 i->iov_offset = skip; \
Al Viro7ce2a912014-11-27 13:59:45 -0500127 } \
Al Viro7ce2a912014-11-27 13:59:45 -0500128}
129
Al Viro62a80672014-04-04 23:12:29 -0400130static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
Al Viro4f18cd32014-02-05 19:11:33 -0500131 struct iov_iter *i)
132{
133 size_t skip, copy, left, wanted;
134 const struct iovec *iov;
135 char __user *buf;
136 void *kaddr, *from;
137
138 if (unlikely(bytes > i->count))
139 bytes = i->count;
140
141 if (unlikely(!bytes))
142 return 0;
143
144 wanted = bytes;
145 iov = i->iov;
146 skip = i->iov_offset;
147 buf = iov->iov_base + skip;
148 copy = min(bytes, iov->iov_len - skip);
149
Mikulas Patocka3fa6c502016-07-28 15:48:50 -0700150 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
Al Viro4f18cd32014-02-05 19:11:33 -0500151 kaddr = kmap_atomic(page);
152 from = kaddr + offset;
153
154 /* first chunk, usually the only one */
155 left = __copy_to_user_inatomic(buf, from, copy);
156 copy -= left;
157 skip += copy;
158 from += copy;
159 bytes -= copy;
160
161 while (unlikely(!left && bytes)) {
162 iov++;
163 buf = iov->iov_base;
164 copy = min(bytes, iov->iov_len);
165 left = __copy_to_user_inatomic(buf, from, copy);
166 copy -= left;
167 skip = copy;
168 from += copy;
169 bytes -= copy;
170 }
171 if (likely(!bytes)) {
172 kunmap_atomic(kaddr);
173 goto done;
174 }
175 offset = from - kaddr;
176 buf += copy;
177 kunmap_atomic(kaddr);
178 copy = min(bytes, iov->iov_len - skip);
179 }
180 /* Too bad - revert to non-atomic kmap */
Mikulas Patocka3fa6c502016-07-28 15:48:50 -0700181
Al Viro4f18cd32014-02-05 19:11:33 -0500182 kaddr = kmap(page);
183 from = kaddr + offset;
184 left = __copy_to_user(buf, from, copy);
185 copy -= left;
186 skip += copy;
187 from += copy;
188 bytes -= copy;
189 while (unlikely(!left && bytes)) {
190 iov++;
191 buf = iov->iov_base;
192 copy = min(bytes, iov->iov_len);
193 left = __copy_to_user(buf, from, copy);
194 copy -= left;
195 skip = copy;
196 from += copy;
197 bytes -= copy;
198 }
199 kunmap(page);
Mikulas Patocka3fa6c502016-07-28 15:48:50 -0700200
Al Viro4f18cd32014-02-05 19:11:33 -0500201done:
Al Viro81055e52014-04-04 19:23:46 -0400202 if (skip == iov->iov_len) {
203 iov++;
204 skip = 0;
205 }
Al Viro4f18cd32014-02-05 19:11:33 -0500206 i->count -= wanted - bytes;
207 i->nr_segs -= iov - i->iov;
208 i->iov = iov;
209 i->iov_offset = skip;
210 return wanted - bytes;
211}
Al Viro4f18cd32014-02-05 19:11:33 -0500212
Al Viro62a80672014-04-04 23:12:29 -0400213static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
Al Virof0d1bec2014-04-03 15:05:18 -0400214 struct iov_iter *i)
215{
216 size_t skip, copy, left, wanted;
217 const struct iovec *iov;
218 char __user *buf;
219 void *kaddr, *to;
220
221 if (unlikely(bytes > i->count))
222 bytes = i->count;
223
224 if (unlikely(!bytes))
225 return 0;
226
227 wanted = bytes;
228 iov = i->iov;
229 skip = i->iov_offset;
230 buf = iov->iov_base + skip;
231 copy = min(bytes, iov->iov_len - skip);
232
Mikulas Patocka3fa6c502016-07-28 15:48:50 -0700233 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
Al Virof0d1bec2014-04-03 15:05:18 -0400234 kaddr = kmap_atomic(page);
235 to = kaddr + offset;
236
237 /* first chunk, usually the only one */
238 left = __copy_from_user_inatomic(to, buf, copy);
239 copy -= left;
240 skip += copy;
241 to += copy;
242 bytes -= copy;
243
244 while (unlikely(!left && bytes)) {
245 iov++;
246 buf = iov->iov_base;
247 copy = min(bytes, iov->iov_len);
248 left = __copy_from_user_inatomic(to, buf, copy);
249 copy -= left;
250 skip = copy;
251 to += copy;
252 bytes -= copy;
253 }
254 if (likely(!bytes)) {
255 kunmap_atomic(kaddr);
256 goto done;
257 }
258 offset = to - kaddr;
259 buf += copy;
260 kunmap_atomic(kaddr);
261 copy = min(bytes, iov->iov_len - skip);
262 }
263 /* Too bad - revert to non-atomic kmap */
Mikulas Patocka3fa6c502016-07-28 15:48:50 -0700264
Al Virof0d1bec2014-04-03 15:05:18 -0400265 kaddr = kmap(page);
266 to = kaddr + offset;
267 left = __copy_from_user(to, buf, copy);
268 copy -= left;
269 skip += copy;
270 to += copy;
271 bytes -= copy;
272 while (unlikely(!left && bytes)) {
273 iov++;
274 buf = iov->iov_base;
275 copy = min(bytes, iov->iov_len);
276 left = __copy_from_user(to, buf, copy);
277 copy -= left;
278 skip = copy;
279 to += copy;
280 bytes -= copy;
281 }
282 kunmap(page);
Mikulas Patocka3fa6c502016-07-28 15:48:50 -0700283
Al Virof0d1bec2014-04-03 15:05:18 -0400284done:
Al Viro81055e52014-04-04 19:23:46 -0400285 if (skip == iov->iov_len) {
286 iov++;
287 skip = 0;
288 }
Al Virof0d1bec2014-04-03 15:05:18 -0400289 i->count -= wanted - bytes;
290 i->nr_segs -= iov - i->iov;
291 i->iov = iov;
292 i->iov_offset = skip;
293 return wanted - bytes;
294}
Al Virof0d1bec2014-04-03 15:05:18 -0400295
Al Viro241699c2016-09-22 16:33:12 -0400296#ifdef PIPE_PARANOIA
297static bool sanity(const struct iov_iter *i)
298{
299 struct pipe_inode_info *pipe = i->pipe;
300 int idx = i->idx;
301 int next = pipe->curbuf + pipe->nrbufs;
302 if (i->iov_offset) {
303 struct pipe_buffer *p;
304 if (unlikely(!pipe->nrbufs))
305 goto Bad; // pipe must be non-empty
306 if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
307 goto Bad; // must be at the last buffer...
308
309 p = &pipe->bufs[idx];
310 if (unlikely(p->offset + p->len != i->iov_offset))
311 goto Bad; // ... at the end of segment
312 } else {
313 if (idx != (next & (pipe->buffers - 1)))
314 goto Bad; // must be right after the last buffer
315 }
316 return true;
317Bad:
318 printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
319 printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
320 pipe->curbuf, pipe->nrbufs, pipe->buffers);
321 for (idx = 0; idx < pipe->buffers; idx++)
322 printk(KERN_ERR "[%p %p %d %d]\n",
323 pipe->bufs[idx].ops,
324 pipe->bufs[idx].page,
325 pipe->bufs[idx].offset,
326 pipe->bufs[idx].len);
327 WARN_ON(1);
328 return false;
329}
330#else
331#define sanity(i) true
332#endif
333
334static inline int next_idx(int idx, struct pipe_inode_info *pipe)
335{
336 return (idx + 1) & (pipe->buffers - 1);
337}
338
339static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
340 struct iov_iter *i)
341{
342 struct pipe_inode_info *pipe = i->pipe;
343 struct pipe_buffer *buf;
344 size_t off;
345 int idx;
346
347 if (unlikely(bytes > i->count))
348 bytes = i->count;
349
350 if (unlikely(!bytes))
351 return 0;
352
353 if (!sanity(i))
354 return 0;
355
356 off = i->iov_offset;
357 idx = i->idx;
358 buf = &pipe->bufs[idx];
359 if (off) {
360 if (offset == off && buf->page == page) {
361 /* merge with the last one */
362 buf->len += bytes;
363 i->iov_offset += bytes;
364 goto out;
365 }
366 idx = next_idx(idx, pipe);
367 buf = &pipe->bufs[idx];
368 }
369 if (idx == pipe->curbuf && pipe->nrbufs)
370 return 0;
371 pipe->nrbufs++;
372 buf->ops = &page_cache_pipe_buf_ops;
373 get_page(buf->page = page);
374 buf->offset = offset;
375 buf->len = bytes;
376 i->iov_offset = offset + bytes;
377 i->idx = idx;
378out:
379 i->count -= bytes;
380 return bytes;
381}
382
Al Viro4f18cd32014-02-05 19:11:33 -0500383/*
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400384 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
385 * bytes. For each iovec, fault in each page that constitutes the iovec.
386 *
387 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
388 * because it is an invalid address).
389 */
Al Virod4690f12016-09-16 00:11:45 +0100390int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400391{
392 size_t skip = i->iov_offset;
393 const struct iovec *iov;
394 int err;
395 struct iovec v;
396
397 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
398 iterate_iovec(i, bytes, v, iov, skip, ({
399 err = fault_in_multipages_readable(v.iov_base,
400 v.iov_len);
401 if (unlikely(err))
402 return err;
403 0;}))
404 }
405 return 0;
406}
Al Virod4690f12016-09-16 00:11:45 +0100407EXPORT_SYMBOL(iov_iter_fault_in_readable);
Anton Altaparmakov171a0202015-03-11 10:43:31 -0400408
Al Viro71d8e532014-03-05 19:28:09 -0500409void iov_iter_init(struct iov_iter *i, int direction,
410 const struct iovec *iov, unsigned long nr_segs,
411 size_t count)
412{
413 /* It will get better. Eventually... */
Al Viroa2804552014-11-27 14:48:42 -0500414 if (segment_eq(get_fs(), KERNEL_DS)) {
Al Viro62a80672014-04-04 23:12:29 -0400415 direction |= ITER_KVEC;
Al Viroa2804552014-11-27 14:48:42 -0500416 i->type = direction;
417 i->kvec = (struct kvec *)iov;
418 } else {
419 i->type = direction;
420 i->iov = iov;
421 }
Al Viro71d8e532014-03-05 19:28:09 -0500422 i->nr_segs = nr_segs;
423 i->iov_offset = 0;
424 i->count = count;
425}
426EXPORT_SYMBOL(iov_iter_init);
Al Viro7b2c99d2014-03-15 04:05:57 -0400427
Al Viro62a80672014-04-04 23:12:29 -0400428static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
429{
430 char *from = kmap_atomic(page);
431 memcpy(to, from + offset, len);
432 kunmap_atomic(from);
433}
434
Al Viro36f7a8a2015-12-06 16:49:22 -0500435static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
Al Viro62a80672014-04-04 23:12:29 -0400436{
437 char *to = kmap_atomic(page);
438 memcpy(to + offset, from, len);
439 kunmap_atomic(to);
440}
441
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400442static void memzero_page(struct page *page, size_t offset, size_t len)
443{
444 char *addr = kmap_atomic(page);
445 memset(addr + offset, 0, len);
446 kunmap_atomic(addr);
447}
448
Al Viro241699c2016-09-22 16:33:12 -0400449static inline bool allocated(struct pipe_buffer *buf)
450{
451 return buf->ops == &default_pipe_buf_ops;
452}
453
454static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
455{
456 size_t off = i->iov_offset;
457 int idx = i->idx;
458 if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
459 idx = next_idx(idx, i->pipe);
460 off = 0;
461 }
462 *idxp = idx;
463 *offp = off;
464}
465
466static size_t push_pipe(struct iov_iter *i, size_t size,
467 int *idxp, size_t *offp)
468{
469 struct pipe_inode_info *pipe = i->pipe;
470 size_t off;
471 int idx;
472 ssize_t left;
473
474 if (unlikely(size > i->count))
475 size = i->count;
476 if (unlikely(!size))
477 return 0;
478
479 left = size;
480 data_start(i, &idx, &off);
481 *idxp = idx;
482 *offp = off;
483 if (off) {
484 left -= PAGE_SIZE - off;
485 if (left <= 0) {
486 pipe->bufs[idx].len += size;
487 return size;
488 }
489 pipe->bufs[idx].len = PAGE_SIZE;
490 idx = next_idx(idx, pipe);
491 }
492 while (idx != pipe->curbuf || !pipe->nrbufs) {
493 struct page *page = alloc_page(GFP_USER);
494 if (!page)
495 break;
496 pipe->nrbufs++;
497 pipe->bufs[idx].ops = &default_pipe_buf_ops;
498 pipe->bufs[idx].page = page;
499 pipe->bufs[idx].offset = 0;
500 if (left <= PAGE_SIZE) {
501 pipe->bufs[idx].len = left;
502 return size;
503 }
504 pipe->bufs[idx].len = PAGE_SIZE;
505 left -= PAGE_SIZE;
506 idx = next_idx(idx, pipe);
507 }
508 return size - left;
509}
510
511static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
512 struct iov_iter *i)
513{
514 struct pipe_inode_info *pipe = i->pipe;
515 size_t n, off;
516 int idx;
517
518 if (!sanity(i))
519 return 0;
520
521 bytes = n = push_pipe(i, bytes, &idx, &off);
522 if (unlikely(!n))
523 return 0;
524 for ( ; n; idx = next_idx(idx, pipe), off = 0) {
525 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
526 memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
527 i->idx = idx;
528 i->iov_offset = off + chunk;
529 n -= chunk;
530 addr += chunk;
531 }
532 i->count -= bytes;
533 return bytes;
534}
535
Al Viro36f7a8a2015-12-06 16:49:22 -0500536size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
Al Viro62a80672014-04-04 23:12:29 -0400537{
Al Viro36f7a8a2015-12-06 16:49:22 -0500538 const char *from = addr;
Al Viro241699c2016-09-22 16:33:12 -0400539 if (unlikely(i->type & ITER_PIPE))
540 return copy_pipe_to_iter(addr, bytes, i);
Al Viro3d4d3e42014-11-27 14:28:06 -0500541 iterate_and_advance(i, bytes, v,
542 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
543 v.iov_len),
544 memcpy_to_page(v.bv_page, v.bv_offset,
Al Viroa2804552014-11-27 14:48:42 -0500545 (from += v.bv_len) - v.bv_len, v.bv_len),
546 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
Al Viro3d4d3e42014-11-27 14:28:06 -0500547 )
Al Viro62a80672014-04-04 23:12:29 -0400548
Al Viro3d4d3e42014-11-27 14:28:06 -0500549 return bytes;
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400550}
551EXPORT_SYMBOL(copy_to_iter);
552
553size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
554{
Al Viro0dbca9a2014-11-27 14:26:43 -0500555 char *to = addr;
Al Viro241699c2016-09-22 16:33:12 -0400556 if (unlikely(i->type & ITER_PIPE)) {
557 WARN_ON(1);
558 return 0;
559 }
Al Viro0dbca9a2014-11-27 14:26:43 -0500560 iterate_and_advance(i, bytes, v,
561 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
562 v.iov_len),
563 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
Al Viroa2804552014-11-27 14:48:42 -0500564 v.bv_offset, v.bv_len),
565 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
Al Viro0dbca9a2014-11-27 14:26:43 -0500566 )
567
568 return bytes;
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400569}
570EXPORT_SYMBOL(copy_from_iter);
571
Al Viroaa583092014-11-27 20:27:08 -0500572size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
573{
574 char *to = addr;
Al Viro241699c2016-09-22 16:33:12 -0400575 if (unlikely(i->type & ITER_PIPE)) {
576 WARN_ON(1);
577 return 0;
578 }
Al Viroaa583092014-11-27 20:27:08 -0500579 iterate_and_advance(i, bytes, v,
580 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
581 v.iov_base, v.iov_len),
582 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
583 v.bv_offset, v.bv_len),
584 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
585 )
586
587 return bytes;
588}
589EXPORT_SYMBOL(copy_from_iter_nocache);
590
Al Virod2715242014-11-27 14:22:37 -0500591size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
592 struct iov_iter *i)
593{
594 if (i->type & (ITER_BVEC|ITER_KVEC)) {
595 void *kaddr = kmap_atomic(page);
596 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
597 kunmap_atomic(kaddr);
598 return wanted;
Al Viro241699c2016-09-22 16:33:12 -0400599 } else if (likely(!(i->type & ITER_PIPE)))
Al Virod2715242014-11-27 14:22:37 -0500600 return copy_page_to_iter_iovec(page, offset, bytes, i);
Al Viro241699c2016-09-22 16:33:12 -0400601 else
602 return copy_page_to_iter_pipe(page, offset, bytes, i);
Al Virod2715242014-11-27 14:22:37 -0500603}
604EXPORT_SYMBOL(copy_page_to_iter);
605
606size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
607 struct iov_iter *i)
608{
Al Viro241699c2016-09-22 16:33:12 -0400609 if (unlikely(i->type & ITER_PIPE)) {
610 WARN_ON(1);
611 return 0;
612 }
Al Viroa2804552014-11-27 14:48:42 -0500613 if (i->type & (ITER_BVEC|ITER_KVEC)) {
Al Virod2715242014-11-27 14:22:37 -0500614 void *kaddr = kmap_atomic(page);
615 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
616 kunmap_atomic(kaddr);
617 return wanted;
618 } else
619 return copy_page_from_iter_iovec(page, offset, bytes, i);
620}
621EXPORT_SYMBOL(copy_page_from_iter);
622
Al Viro241699c2016-09-22 16:33:12 -0400623static size_t pipe_zero(size_t bytes, struct iov_iter *i)
624{
625 struct pipe_inode_info *pipe = i->pipe;
626 size_t n, off;
627 int idx;
628
629 if (!sanity(i))
630 return 0;
631
632 bytes = n = push_pipe(i, bytes, &idx, &off);
633 if (unlikely(!n))
634 return 0;
635
636 for ( ; n; idx = next_idx(idx, pipe), off = 0) {
637 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
638 memzero_page(pipe->bufs[idx].page, off, chunk);
639 i->idx = idx;
640 i->iov_offset = off + chunk;
641 n -= chunk;
642 }
643 i->count -= bytes;
644 return bytes;
645}
646
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400647size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
648{
Al Viro241699c2016-09-22 16:33:12 -0400649 if (unlikely(i->type & ITER_PIPE))
650 return pipe_zero(bytes, i);
Al Viro8442fa42014-11-27 14:18:54 -0500651 iterate_and_advance(i, bytes, v,
652 __clear_user(v.iov_base, v.iov_len),
Al Viroa2804552014-11-27 14:48:42 -0500653 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
654 memset(v.iov_base, 0, v.iov_len)
Al Viro8442fa42014-11-27 14:18:54 -0500655 )
656
657 return bytes;
Matthew Wilcoxc35e0242014-08-01 09:27:22 -0400658}
659EXPORT_SYMBOL(iov_iter_zero);
660
Al Viro62a80672014-04-04 23:12:29 -0400661size_t iov_iter_copy_from_user_atomic(struct page *page,
662 struct iov_iter *i, unsigned long offset, size_t bytes)
663{
Al Viro04a31162014-11-27 13:51:41 -0500664 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
Al Viro241699c2016-09-22 16:33:12 -0400665 if (unlikely(i->type & ITER_PIPE)) {
666 kunmap_atomic(kaddr);
667 WARN_ON(1);
668 return 0;
669 }
Al Viro04a31162014-11-27 13:51:41 -0500670 iterate_all_kinds(i, bytes, v,
671 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
672 v.iov_base, v.iov_len),
673 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
Al Viroa2804552014-11-27 14:48:42 -0500674 v.bv_offset, v.bv_len),
675 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
Al Viro04a31162014-11-27 13:51:41 -0500676 )
677 kunmap_atomic(kaddr);
678 return bytes;
Al Viro62a80672014-04-04 23:12:29 -0400679}
680EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
681
Al Viro241699c2016-09-22 16:33:12 -0400682static void pipe_advance(struct iov_iter *i, size_t size)
683{
684 struct pipe_inode_info *pipe = i->pipe;
685 struct pipe_buffer *buf;
686 int idx = i->idx;
687 size_t off = i->iov_offset;
688
689 if (unlikely(i->count < size))
690 size = i->count;
691
692 if (size) {
693 if (off) /* make it relative to the beginning of buffer */
694 size += off - pipe->bufs[idx].offset;
695 while (1) {
696 buf = &pipe->bufs[idx];
697 if (size <= buf->len)
698 break;
699 size -= buf->len;
700 idx = next_idx(idx, pipe);
701 }
702 buf->len = size;
703 i->idx = idx;
704 off = i->iov_offset = buf->offset + size;
705 }
706 if (off)
707 idx = next_idx(idx, pipe);
708 if (pipe->nrbufs) {
709 int unused = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
710 /* [curbuf,unused) is in use. Free [idx,unused) */
711 while (idx != unused) {
Miklos Szeredia7796382016-09-27 10:45:12 +0200712 pipe_buf_release(pipe, &pipe->bufs[idx]);
Al Viro241699c2016-09-22 16:33:12 -0400713 idx = next_idx(idx, pipe);
714 pipe->nrbufs--;
715 }
716 }
717}
718
Al Viro62a80672014-04-04 23:12:29 -0400719void iov_iter_advance(struct iov_iter *i, size_t size)
720{
Al Viro241699c2016-09-22 16:33:12 -0400721 if (unlikely(i->type & ITER_PIPE)) {
722 pipe_advance(i, size);
723 return;
724 }
Al Viroa2804552014-11-27 14:48:42 -0500725 iterate_and_advance(i, size, v, 0, 0, 0)
Al Viro62a80672014-04-04 23:12:29 -0400726}
727EXPORT_SYMBOL(iov_iter_advance);
728
729/*
730 * Return the count of just the current iov_iter segment.
731 */
732size_t iov_iter_single_seg_count(const struct iov_iter *i)
733{
Al Viro241699c2016-09-22 16:33:12 -0400734 if (unlikely(i->type & ITER_PIPE))
735 return i->count; // it is a silly place, anyway
Al Viro62a80672014-04-04 23:12:29 -0400736 if (i->nr_segs == 1)
737 return i->count;
738 else if (i->type & ITER_BVEC)
Al Viro62a80672014-04-04 23:12:29 -0400739 return min(i->count, i->bvec->bv_len - i->iov_offset);
Paul Mackerrasad0eab92014-11-13 20:15:23 +1100740 else
741 return min(i->count, i->iov->iov_len - i->iov_offset);
Al Viro62a80672014-04-04 23:12:29 -0400742}
743EXPORT_SYMBOL(iov_iter_single_seg_count);
744
Al Viroabb78f82014-11-24 14:46:11 -0500745void iov_iter_kvec(struct iov_iter *i, int direction,
Al Viro05afcb72015-01-23 01:08:07 -0500746 const struct kvec *kvec, unsigned long nr_segs,
Al Viroabb78f82014-11-24 14:46:11 -0500747 size_t count)
748{
749 BUG_ON(!(direction & ITER_KVEC));
750 i->type = direction;
Al Viro05afcb72015-01-23 01:08:07 -0500751 i->kvec = kvec;
Al Viroabb78f82014-11-24 14:46:11 -0500752 i->nr_segs = nr_segs;
753 i->iov_offset = 0;
754 i->count = count;
755}
756EXPORT_SYMBOL(iov_iter_kvec);
757
Al Viro05afcb72015-01-23 01:08:07 -0500758void iov_iter_bvec(struct iov_iter *i, int direction,
759 const struct bio_vec *bvec, unsigned long nr_segs,
760 size_t count)
761{
762 BUG_ON(!(direction & ITER_BVEC));
763 i->type = direction;
764 i->bvec = bvec;
765 i->nr_segs = nr_segs;
766 i->iov_offset = 0;
767 i->count = count;
768}
769EXPORT_SYMBOL(iov_iter_bvec);
770
Al Viro241699c2016-09-22 16:33:12 -0400771void iov_iter_pipe(struct iov_iter *i, int direction,
772 struct pipe_inode_info *pipe,
773 size_t count)
774{
775 BUG_ON(direction != ITER_PIPE);
776 i->type = direction;
777 i->pipe = pipe;
778 i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
779 i->iov_offset = 0;
780 i->count = count;
781}
782EXPORT_SYMBOL(iov_iter_pipe);
783
Al Viro62a80672014-04-04 23:12:29 -0400784unsigned long iov_iter_alignment(const struct iov_iter *i)
785{
Al Viro04a31162014-11-27 13:51:41 -0500786 unsigned long res = 0;
787 size_t size = i->count;
788
789 if (!size)
790 return 0;
791
Al Viro241699c2016-09-22 16:33:12 -0400792 if (unlikely(i->type & ITER_PIPE)) {
793 if (i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
794 return size | i->iov_offset;
795 return size;
796 }
Al Viro04a31162014-11-27 13:51:41 -0500797 iterate_all_kinds(i, size, v,
798 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
Al Viroa2804552014-11-27 14:48:42 -0500799 res |= v.bv_offset | v.bv_len,
800 res |= (unsigned long)v.iov_base | v.iov_len
Al Viro04a31162014-11-27 13:51:41 -0500801 )
802 return res;
Al Viro62a80672014-04-04 23:12:29 -0400803}
804EXPORT_SYMBOL(iov_iter_alignment);
805
Al Viro357f4352016-04-08 19:05:19 -0400806unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
807{
808 unsigned long res = 0;
809 size_t size = i->count;
810 if (!size)
811 return 0;
812
Al Viro241699c2016-09-22 16:33:12 -0400813 if (unlikely(i->type & ITER_PIPE)) {
814 WARN_ON(1);
815 return ~0U;
816 }
817
Al Viro357f4352016-04-08 19:05:19 -0400818 iterate_all_kinds(i, size, v,
819 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
820 (size != v.iov_len ? size : 0), 0),
821 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
822 (size != v.bv_len ? size : 0)),
823 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
824 (size != v.iov_len ? size : 0))
825 );
826 return res;
827}
828EXPORT_SYMBOL(iov_iter_gap_alignment);
829
Al Viro241699c2016-09-22 16:33:12 -0400830static inline size_t __pipe_get_pages(struct iov_iter *i,
831 size_t maxsize,
832 struct page **pages,
833 int idx,
834 size_t *start)
835{
836 struct pipe_inode_info *pipe = i->pipe;
837 size_t n = push_pipe(i, maxsize, &idx, start);
838 if (!n)
839 return -EFAULT;
840
841 maxsize = n;
842 n += *start;
843 while (n >= PAGE_SIZE) {
844 get_page(*pages++ = pipe->bufs[idx].page);
845 idx = next_idx(idx, pipe);
846 n -= PAGE_SIZE;
847 }
848
849 return maxsize;
850}
851
852static ssize_t pipe_get_pages(struct iov_iter *i,
853 struct page **pages, size_t maxsize, unsigned maxpages,
854 size_t *start)
855{
856 unsigned npages;
857 size_t capacity;
858 int idx;
859
860 if (!sanity(i))
861 return -EFAULT;
862
863 data_start(i, &idx, start);
864 /* some of this one + all after this one */
865 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
866 capacity = min(npages,maxpages) * PAGE_SIZE - *start;
867
868 return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
869}
870
Al Viro62a80672014-04-04 23:12:29 -0400871ssize_t iov_iter_get_pages(struct iov_iter *i,
Miklos Szeredi2c809292014-09-24 17:09:11 +0200872 struct page **pages, size_t maxsize, unsigned maxpages,
Al Viro62a80672014-04-04 23:12:29 -0400873 size_t *start)
874{
Al Viroe5393fa2014-11-27 14:12:09 -0500875 if (maxsize > i->count)
876 maxsize = i->count;
877
878 if (!maxsize)
879 return 0;
880
Al Viro241699c2016-09-22 16:33:12 -0400881 if (unlikely(i->type & ITER_PIPE))
882 return pipe_get_pages(i, pages, maxsize, maxpages, start);
Al Viroe5393fa2014-11-27 14:12:09 -0500883 iterate_all_kinds(i, maxsize, v, ({
884 unsigned long addr = (unsigned long)v.iov_base;
885 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
886 int n;
887 int res;
888
889 if (len > maxpages * PAGE_SIZE)
890 len = maxpages * PAGE_SIZE;
891 addr &= ~(PAGE_SIZE - 1);
892 n = DIV_ROUND_UP(len, PAGE_SIZE);
893 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
894 if (unlikely(res < 0))
895 return res;
896 return (res == n ? len : res * PAGE_SIZE) - *start;
897 0;}),({
898 /* can't be more than PAGE_SIZE */
899 *start = v.bv_offset;
900 get_page(*pages = v.bv_page);
901 return v.bv_len;
Al Viroa2804552014-11-27 14:48:42 -0500902 }),({
903 return -EFAULT;
Al Viroe5393fa2014-11-27 14:12:09 -0500904 })
905 )
906 return 0;
Al Viro62a80672014-04-04 23:12:29 -0400907}
908EXPORT_SYMBOL(iov_iter_get_pages);
909
Al Viro1b17f1f2014-11-27 14:14:31 -0500910static struct page **get_pages_array(size_t n)
911{
912 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
913 if (!p)
914 p = vmalloc(n * sizeof(struct page *));
915 return p;
916}
917
Al Viro241699c2016-09-22 16:33:12 -0400918static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
919 struct page ***pages, size_t maxsize,
920 size_t *start)
921{
922 struct page **p;
923 size_t n;
924 int idx;
925 int npages;
926
927 if (!sanity(i))
928 return -EFAULT;
929
930 data_start(i, &idx, start);
931 /* some of this one + all after this one */
932 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
933 n = npages * PAGE_SIZE - *start;
934 if (maxsize > n)
935 maxsize = n;
936 else
937 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
938 p = get_pages_array(npages);
939 if (!p)
940 return -ENOMEM;
941 n = __pipe_get_pages(i, maxsize, p, idx, start);
942 if (n > 0)
943 *pages = p;
944 else
945 kvfree(p);
946 return n;
947}
948
Al Viro62a80672014-04-04 23:12:29 -0400949ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
950 struct page ***pages, size_t maxsize,
951 size_t *start)
952{
Al Viro1b17f1f2014-11-27 14:14:31 -0500953 struct page **p;
954
955 if (maxsize > i->count)
956 maxsize = i->count;
957
958 if (!maxsize)
959 return 0;
960
Al Viro241699c2016-09-22 16:33:12 -0400961 if (unlikely(i->type & ITER_PIPE))
962 return pipe_get_pages_alloc(i, pages, maxsize, start);
Al Viro1b17f1f2014-11-27 14:14:31 -0500963 iterate_all_kinds(i, maxsize, v, ({
964 unsigned long addr = (unsigned long)v.iov_base;
965 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
966 int n;
967 int res;
968
969 addr &= ~(PAGE_SIZE - 1);
970 n = DIV_ROUND_UP(len, PAGE_SIZE);
971 p = get_pages_array(n);
972 if (!p)
973 return -ENOMEM;
974 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
975 if (unlikely(res < 0)) {
976 kvfree(p);
977 return res;
978 }
979 *pages = p;
980 return (res == n ? len : res * PAGE_SIZE) - *start;
981 0;}),({
982 /* can't be more than PAGE_SIZE */
983 *start = v.bv_offset;
984 *pages = p = get_pages_array(1);
985 if (!p)
986 return -ENOMEM;
987 get_page(*p = v.bv_page);
988 return v.bv_len;
Al Viroa2804552014-11-27 14:48:42 -0500989 }),({
990 return -EFAULT;
Al Viro1b17f1f2014-11-27 14:14:31 -0500991 })
992 )
993 return 0;
Al Viro62a80672014-04-04 23:12:29 -0400994}
995EXPORT_SYMBOL(iov_iter_get_pages_alloc);
996
Al Viroa604ec72014-11-24 01:08:00 -0500997size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
998 struct iov_iter *i)
999{
1000 char *to = addr;
1001 __wsum sum, next;
1002 size_t off = 0;
Al Viroa604ec72014-11-24 01:08:00 -05001003 sum = *csum;
Al Viro241699c2016-09-22 16:33:12 -04001004 if (unlikely(i->type & ITER_PIPE)) {
1005 WARN_ON(1);
1006 return 0;
1007 }
Al Viroa604ec72014-11-24 01:08:00 -05001008 iterate_and_advance(i, bytes, v, ({
1009 int err = 0;
1010 next = csum_and_copy_from_user(v.iov_base,
1011 (to += v.iov_len) - v.iov_len,
1012 v.iov_len, 0, &err);
1013 if (!err) {
1014 sum = csum_block_add(sum, next, off);
1015 off += v.iov_len;
1016 }
1017 err ? v.iov_len : 0;
1018 }), ({
1019 char *p = kmap_atomic(v.bv_page);
1020 next = csum_partial_copy_nocheck(p + v.bv_offset,
1021 (to += v.bv_len) - v.bv_len,
1022 v.bv_len, 0);
1023 kunmap_atomic(p);
1024 sum = csum_block_add(sum, next, off);
1025 off += v.bv_len;
1026 }),({
1027 next = csum_partial_copy_nocheck(v.iov_base,
1028 (to += v.iov_len) - v.iov_len,
1029 v.iov_len, 0);
1030 sum = csum_block_add(sum, next, off);
1031 off += v.iov_len;
1032 })
1033 )
1034 *csum = sum;
1035 return bytes;
1036}
1037EXPORT_SYMBOL(csum_and_copy_from_iter);
1038
Al Viro36f7a8a2015-12-06 16:49:22 -05001039size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
Al Viroa604ec72014-11-24 01:08:00 -05001040 struct iov_iter *i)
1041{
Al Viro36f7a8a2015-12-06 16:49:22 -05001042 const char *from = addr;
Al Viroa604ec72014-11-24 01:08:00 -05001043 __wsum sum, next;
1044 size_t off = 0;
Al Viroa604ec72014-11-24 01:08:00 -05001045 sum = *csum;
Al Viro241699c2016-09-22 16:33:12 -04001046 if (unlikely(i->type & ITER_PIPE)) {
1047 WARN_ON(1); /* for now */
1048 return 0;
1049 }
Al Viroa604ec72014-11-24 01:08:00 -05001050 iterate_and_advance(i, bytes, v, ({
1051 int err = 0;
1052 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1053 v.iov_base,
1054 v.iov_len, 0, &err);
1055 if (!err) {
1056 sum = csum_block_add(sum, next, off);
1057 off += v.iov_len;
1058 }
1059 err ? v.iov_len : 0;
1060 }), ({
1061 char *p = kmap_atomic(v.bv_page);
1062 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
1063 p + v.bv_offset,
1064 v.bv_len, 0);
1065 kunmap_atomic(p);
1066 sum = csum_block_add(sum, next, off);
1067 off += v.bv_len;
1068 }),({
1069 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
1070 v.iov_base,
1071 v.iov_len, 0);
1072 sum = csum_block_add(sum, next, off);
1073 off += v.iov_len;
1074 })
1075 )
1076 *csum = sum;
1077 return bytes;
1078}
1079EXPORT_SYMBOL(csum_and_copy_to_iter);
1080
Al Viro62a80672014-04-04 23:12:29 -04001081int iov_iter_npages(const struct iov_iter *i, int maxpages)
1082{
Al Viroe0f2dc42014-11-27 14:09:46 -05001083 size_t size = i->count;
1084 int npages = 0;
1085
1086 if (!size)
1087 return 0;
1088
Al Viro241699c2016-09-22 16:33:12 -04001089 if (unlikely(i->type & ITER_PIPE)) {
1090 struct pipe_inode_info *pipe = i->pipe;
1091 size_t off;
1092 int idx;
1093
1094 if (!sanity(i))
1095 return 0;
1096
1097 data_start(i, &idx, &off);
1098 /* some of this one + all after this one */
1099 npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
1100 if (npages >= maxpages)
1101 return maxpages;
1102 } else iterate_all_kinds(i, size, v, ({
Al Viroe0f2dc42014-11-27 14:09:46 -05001103 unsigned long p = (unsigned long)v.iov_base;
1104 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1105 - p / PAGE_SIZE;
1106 if (npages >= maxpages)
1107 return maxpages;
1108 0;}),({
1109 npages++;
1110 if (npages >= maxpages)
1111 return maxpages;
Al Viroa2804552014-11-27 14:48:42 -05001112 }),({
1113 unsigned long p = (unsigned long)v.iov_base;
1114 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1115 - p / PAGE_SIZE;
1116 if (npages >= maxpages)
1117 return maxpages;
Al Viroe0f2dc42014-11-27 14:09:46 -05001118 })
1119 )
1120 return npages;
Al Viro62a80672014-04-04 23:12:29 -04001121}
Al Virof67da302014-03-19 01:16:16 -04001122EXPORT_SYMBOL(iov_iter_npages);
Al Viro4b8164b2015-01-31 20:08:47 -05001123
1124const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1125{
1126 *new = *old;
Al Viro241699c2016-09-22 16:33:12 -04001127 if (unlikely(new->type & ITER_PIPE)) {
1128 WARN_ON(1);
1129 return NULL;
1130 }
Al Viro4b8164b2015-01-31 20:08:47 -05001131 if (new->type & ITER_BVEC)
1132 return new->bvec = kmemdup(new->bvec,
1133 new->nr_segs * sizeof(struct bio_vec),
1134 flags);
1135 else
1136 /* iovec and kvec have identical layout */
1137 return new->iov = kmemdup(new->iov,
1138 new->nr_segs * sizeof(struct iovec),
1139 flags);
1140}
1141EXPORT_SYMBOL(dup_iter);
Al Virobc917be2015-03-21 17:45:43 -04001142
1143int import_iovec(int type, const struct iovec __user * uvector,
1144 unsigned nr_segs, unsigned fast_segs,
1145 struct iovec **iov, struct iov_iter *i)
1146{
1147 ssize_t n;
1148 struct iovec *p;
1149 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1150 *iov, &p);
1151 if (n < 0) {
1152 if (p != *iov)
1153 kfree(p);
1154 *iov = NULL;
1155 return n;
1156 }
1157 iov_iter_init(i, type, p, nr_segs, n);
1158 *iov = p == *iov ? NULL : p;
1159 return 0;
1160}
1161EXPORT_SYMBOL(import_iovec);
1162
1163#ifdef CONFIG_COMPAT
1164#include <linux/compat.h>
1165
1166int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
1167 unsigned nr_segs, unsigned fast_segs,
1168 struct iovec **iov, struct iov_iter *i)
1169{
1170 ssize_t n;
1171 struct iovec *p;
1172 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1173 *iov, &p);
1174 if (n < 0) {
1175 if (p != *iov)
1176 kfree(p);
1177 *iov = NULL;
1178 return n;
1179 }
1180 iov_iter_init(i, type, p, nr_segs, n);
1181 *iov = p == *iov ? NULL : p;
1182 return 0;
1183}
1184#endif
1185
1186int import_single_range(int rw, void __user *buf, size_t len,
1187 struct iovec *iov, struct iov_iter *i)
1188{
1189 if (len > MAX_RW_COUNT)
1190 len = MAX_RW_COUNT;
1191 if (unlikely(!access_ok(!rw, buf, len)))
1192 return -EFAULT;
1193
1194 iov->iov_base = buf;
1195 iov->iov_len = len;
1196 iov_iter_init(i, rw, iov, 1, len);
1197 return 0;
1198}
Al Viroe1267582015-12-06 20:38:56 -05001199EXPORT_SYMBOL(import_single_range);