blob: 297c7238f7d4094a6ac4ab0dc72e04abb870972f [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Jan Kara8025e5d2015-07-13 11:55:44 -03002#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/err.h>
5#include <linux/mm.h>
6#include <linux/slab.h>
7#include <linux/vmalloc.h>
8#include <linux/pagemap.h>
9#include <linux/sched.h>
10
Jonathan Corbet61f9ec12015-11-05 18:46:23 -080011/**
Jan Kara8025e5d2015-07-13 11:55:44 -030012 * get_vaddr_frames() - map virtual addresses to pfns
13 * @start: starting user address
14 * @nr_frames: number of pages / pfns from start to map
Lorenzo Stoakes7f23b352016-10-13 01:20:15 +010015 * @gup_flags: flags modifying lookup behaviour
Jan Kara8025e5d2015-07-13 11:55:44 -030016 * @vec: structure which receives pages / pfns of the addresses mapped.
17 * It should have space for at least nr_frames entries.
18 *
19 * This function maps virtual addresses from @start and fills @vec structure
20 * with page frame numbers or page pointers to corresponding pages (choice
21 * depends on the type of the vma underlying the virtual address). If @start
22 * belongs to a normal vma, the function grabs reference to each of the pages
23 * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't
24 * touch page structures and the caller must make sure pfns aren't reused for
25 * anything else while he is using them.
26 *
27 * The function returns number of pages mapped which may be less than
28 * @nr_frames. In particular we stop mapping if there are more vmas of
29 * different type underlying the specified range of virtual addresses.
30 * When the function isn't able to map a single page, it returns error.
31 *
32 * This function takes care of grabbing mmap_sem as necessary.
33 */
34int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
Lorenzo Stoakes7f23b352016-10-13 01:20:15 +010035 unsigned int gup_flags, struct frame_vector *vec)
Jan Kara8025e5d2015-07-13 11:55:44 -030036{
37 struct mm_struct *mm = current->mm;
38 struct vm_area_struct *vma;
39 int ret = 0;
40 int err;
41 int locked;
42
43 if (nr_frames == 0)
44 return 0;
45
46 if (WARN_ON_ONCE(nr_frames > vec->nr_allocated))
47 nr_frames = vec->nr_allocated;
48
49 down_read(&mm->mmap_sem);
50 locked = 1;
51 vma = find_vma_intersection(mm, start, start + 1);
52 if (!vma) {
53 ret = -EFAULT;
54 goto out;
55 }
Dan Williamsb7f05542017-11-29 16:10:39 -080056
57 /*
58 * While get_vaddr_frames() could be used for transient (kernel
59 * controlled lifetime) pinning of memory pages all current
60 * users establish long term (userspace controlled lifetime)
61 * page pinning. Treat get_vaddr_frames() like
62 * get_user_pages_longterm() and disallow it for filesystem-dax
63 * mappings.
64 */
65 if (vma_is_fsdax(vma))
66 return -EOPNOTSUPP;
67
Jan Kara8025e5d2015-07-13 11:55:44 -030068 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) {
69 vec->got_ref = true;
70 vec->is_pfns = false;
Dave Hansend4edcf02016-02-12 13:01:56 -080071 ret = get_user_pages_locked(start, nr_frames,
Lorenzo Stoakes3b913172016-10-13 01:20:14 +010072 gup_flags, (struct page **)(vec->ptrs), &locked);
Jan Kara8025e5d2015-07-13 11:55:44 -030073 goto out;
74 }
75
76 vec->got_ref = false;
77 vec->is_pfns = true;
78 do {
79 unsigned long *nums = frame_vector_pfns(vec);
80
81 while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) {
82 err = follow_pfn(vma, start, &nums[ret]);
83 if (err) {
84 if (ret == 0)
85 ret = err;
86 goto out;
87 }
88 start += PAGE_SIZE;
89 ret++;
90 }
91 /*
92 * We stop if we have enough pages or if VMA doesn't completely
93 * cover the tail page.
94 */
95 if (ret >= nr_frames || start < vma->vm_end)
96 break;
97 vma = find_vma_intersection(mm, start, start + 1);
98 } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP));
99out:
100 if (locked)
101 up_read(&mm->mmap_sem);
102 if (!ret)
103 ret = -EFAULT;
104 if (ret > 0)
105 vec->nr_frames = ret;
106 return ret;
107}
108EXPORT_SYMBOL(get_vaddr_frames);
109
110/**
111 * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired
112 * them
113 * @vec: frame vector to put
114 *
115 * Drop references to pages if get_vaddr_frames() acquired them. We also
116 * invalidate the frame vector so that it is prepared for the next call into
117 * get_vaddr_frames().
118 */
119void put_vaddr_frames(struct frame_vector *vec)
120{
121 int i;
122 struct page **pages;
123
124 if (!vec->got_ref)
125 goto out;
126 pages = frame_vector_pages(vec);
127 /*
128 * frame_vector_pages() might needed to do a conversion when
129 * get_vaddr_frames() got pages but vec was later converted to pfns.
130 * But it shouldn't really fail to convert pfns back...
131 */
132 if (WARN_ON(IS_ERR(pages)))
133 goto out;
134 for (i = 0; i < vec->nr_frames; i++)
135 put_page(pages[i]);
136 vec->got_ref = false;
137out:
138 vec->nr_frames = 0;
139}
140EXPORT_SYMBOL(put_vaddr_frames);
141
142/**
143 * frame_vector_to_pages - convert frame vector to contain page pointers
144 * @vec: frame vector to convert
145 *
146 * Convert @vec to contain array of page pointers. If the conversion is
147 * successful, return 0. Otherwise return an error. Note that we do not grab
148 * page references for the page structures.
149 */
150int frame_vector_to_pages(struct frame_vector *vec)
151{
152 int i;
153 unsigned long *nums;
154 struct page **pages;
155
156 if (!vec->is_pfns)
157 return 0;
158 nums = frame_vector_pfns(vec);
159 for (i = 0; i < vec->nr_frames; i++)
160 if (!pfn_valid(nums[i]))
161 return -EINVAL;
162 pages = (struct page **)nums;
163 for (i = 0; i < vec->nr_frames; i++)
164 pages[i] = pfn_to_page(nums[i]);
165 vec->is_pfns = false;
166 return 0;
167}
168EXPORT_SYMBOL(frame_vector_to_pages);
169
170/**
171 * frame_vector_to_pfns - convert frame vector to contain pfns
172 * @vec: frame vector to convert
173 *
174 * Convert @vec to contain array of pfns.
175 */
176void frame_vector_to_pfns(struct frame_vector *vec)
177{
178 int i;
179 unsigned long *nums;
180 struct page **pages;
181
182 if (vec->is_pfns)
183 return;
184 pages = (struct page **)(vec->ptrs);
185 nums = (unsigned long *)pages;
186 for (i = 0; i < vec->nr_frames; i++)
187 nums[i] = page_to_pfn(pages[i]);
188 vec->is_pfns = true;
189}
190EXPORT_SYMBOL(frame_vector_to_pfns);
191
192/**
193 * frame_vector_create() - allocate & initialize structure for pinned pfns
194 * @nr_frames: number of pfns slots we should reserve
195 *
196 * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns
197 * pfns.
198 */
199struct frame_vector *frame_vector_create(unsigned int nr_frames)
200{
201 struct frame_vector *vec;
202 int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames;
203
204 if (WARN_ON_ONCE(nr_frames == 0))
205 return NULL;
206 /*
207 * This is absurdly high. It's here just to avoid strange effects when
208 * arithmetics overflows.
209 */
210 if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2))
211 return NULL;
212 /*
213 * Avoid higher order allocations, use vmalloc instead. It should
214 * be rare anyway.
215 */
Michal Hocko752ade62017-05-08 15:57:27 -0700216 vec = kvmalloc(size, GFP_KERNEL);
Jan Kara8025e5d2015-07-13 11:55:44 -0300217 if (!vec)
218 return NULL;
219 vec->nr_allocated = nr_frames;
220 vec->nr_frames = 0;
221 return vec;
222}
223EXPORT_SYMBOL(frame_vector_create);
224
225/**
226 * frame_vector_destroy() - free memory allocated to carry frame vector
227 * @vec: Frame vector to free
228 *
229 * Free structure allocated by frame_vector_create() to carry frames.
230 */
231void frame_vector_destroy(struct frame_vector *vec)
232{
233 /* Make sure put_vaddr_frames() got called properly... */
234 VM_BUG_ON(vec->nr_frames > 0);
235 kvfree(vec);
236}
237EXPORT_SYMBOL(frame_vector_destroy);