blob: dc3036cdcc6a60ac4e8bea6521279b2cff687bcb [file] [log] [blame]
Andrew Morton16d69262008-07-25 19:44:36 -07001#include <linux/mm.h>
Matt Mackall30992c92006-01-08 01:01:43 -08002#include <linux/slab.h>
3#include <linux/string.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -04004#include <linux/export.h>
Davi Arnaut96840aa2006-03-24 03:18:42 -08005#include <linux/err.h>
Adrian Bunk3b8f14b2008-07-26 15:22:28 -07006#include <linux/sched.h>
Al Viroeb36c582012-05-30 20:17:35 -04007#include <linux/security.h>
Davi Arnaut96840aa2006-03-24 03:18:42 -08008#include <asm/uaccess.h>
Matt Mackall30992c92006-01-08 01:01:43 -08009
Namhyung Kim6038def2011-05-24 17:11:22 -070010#include "internal.h"
11
Steven Rostedta8d154b2009-04-10 09:36:00 -040012#define CREATE_TRACE_POINTS
Steven Rostedtad8d75f2009-04-14 19:39:12 -040013#include <trace/events/kmem.h>
Steven Rostedta8d154b2009-04-10 09:36:00 -040014
Matt Mackall30992c92006-01-08 01:01:43 -080015/**
Matt Mackall30992c92006-01-08 01:01:43 -080016 * kstrdup - allocate space for and copy an existing string
Matt Mackall30992c92006-01-08 01:01:43 -080017 * @s: the string to duplicate
18 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
19 */
20char *kstrdup(const char *s, gfp_t gfp)
21{
22 size_t len;
23 char *buf;
24
25 if (!s)
26 return NULL;
27
28 len = strlen(s) + 1;
Christoph Hellwig1d2c8ee2006-10-04 02:15:25 -070029 buf = kmalloc_track_caller(len, gfp);
Matt Mackall30992c92006-01-08 01:01:43 -080030 if (buf)
31 memcpy(buf, s, len);
32 return buf;
33}
34EXPORT_SYMBOL(kstrdup);
Davi Arnaut96840aa2006-03-24 03:18:42 -080035
Alexey Dobriyan1a2f67b2006-09-30 23:27:20 -070036/**
Jeremy Fitzhardinge1e66df32007-07-17 18:37:02 -070037 * kstrndup - allocate space for and copy an existing string
38 * @s: the string to duplicate
39 * @max: read at most @max chars from @s
40 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
41 */
42char *kstrndup(const char *s, size_t max, gfp_t gfp)
43{
44 size_t len;
45 char *buf;
46
47 if (!s)
48 return NULL;
49
50 len = strnlen(s, max);
51 buf = kmalloc_track_caller(len+1, gfp);
52 if (buf) {
53 memcpy(buf, s, len);
54 buf[len] = '\0';
55 }
56 return buf;
57}
58EXPORT_SYMBOL(kstrndup);
59
60/**
Alexey Dobriyan1a2f67b2006-09-30 23:27:20 -070061 * kmemdup - duplicate region of memory
62 *
63 * @src: memory region to duplicate
64 * @len: memory region length
65 * @gfp: GFP mask to use
66 */
67void *kmemdup(const void *src, size_t len, gfp_t gfp)
68{
69 void *p;
70
Christoph Hellwig1d2c8ee2006-10-04 02:15:25 -070071 p = kmalloc_track_caller(len, gfp);
Alexey Dobriyan1a2f67b2006-09-30 23:27:20 -070072 if (p)
73 memcpy(p, src, len);
74 return p;
75}
76EXPORT_SYMBOL(kmemdup);
77
Christoph Lameteref2ad802007-07-17 04:03:21 -070078/**
Li Zefan610a77e2009-03-31 15:23:16 -070079 * memdup_user - duplicate memory region from user space
80 *
81 * @src: source address in user space
82 * @len: number of bytes to copy
83 *
84 * Returns an ERR_PTR() on failure.
85 */
86void *memdup_user(const void __user *src, size_t len)
87{
88 void *p;
89
90 /*
91 * Always use GFP_KERNEL, since copy_from_user() can sleep and
92 * cause pagefault, which makes it pointless to use GFP_NOFS
93 * or GFP_ATOMIC.
94 */
95 p = kmalloc_track_caller(len, GFP_KERNEL);
96 if (!p)
97 return ERR_PTR(-ENOMEM);
98
99 if (copy_from_user(p, src, len)) {
100 kfree(p);
101 return ERR_PTR(-EFAULT);
102 }
103
104 return p;
105}
106EXPORT_SYMBOL(memdup_user);
107
Ezequiel Garciae21827a2012-08-14 09:55:21 -0300108static __always_inline void *__do_krealloc(const void *p, size_t new_size,
109 gfp_t flags)
Pekka Enberg93bc4e82008-07-26 17:49:33 -0700110{
111 void *ret;
112 size_t ks = 0;
113
Pekka Enberg93bc4e82008-07-26 17:49:33 -0700114 if (p)
115 ks = ksize(p);
116
117 if (ks >= new_size)
118 return (void *)p;
119
120 ret = kmalloc_track_caller(new_size, flags);
121 if (ret && p)
122 memcpy(ret, p, ks);
123
124 return ret;
125}
Ezequiel Garciae21827a2012-08-14 09:55:21 -0300126
127/**
128 * __krealloc - like krealloc() but don't free @p.
129 * @p: object to reallocate memory for.
130 * @new_size: how many bytes of memory are required.
131 * @flags: the type of memory to allocate.
132 *
133 * This function is like krealloc() except it never frees the originally
134 * allocated buffer. Use this if you don't want to free the buffer immediately
135 * like, for example, with RCU.
136 */
137void *__krealloc(const void *p, size_t new_size, gfp_t flags)
138{
139 if (unlikely(!new_size))
140 return ZERO_SIZE_PTR;
141
142 return __do_krealloc(p, new_size, flags);
143
144}
Pekka Enberg93bc4e82008-07-26 17:49:33 -0700145EXPORT_SYMBOL(__krealloc);
146
147/**
Christoph Lameteref2ad802007-07-17 04:03:21 -0700148 * krealloc - reallocate memory. The contents will remain unchanged.
149 * @p: object to reallocate memory for.
150 * @new_size: how many bytes of memory are required.
151 * @flags: the type of memory to allocate.
152 *
153 * The contents of the object pointed to are preserved up to the
154 * lesser of the new and old sizes. If @p is %NULL, krealloc()
155 * behaves exactly like kmalloc(). If @size is 0 and @p is not a
156 * %NULL pointer, the object pointed to is freed.
157 */
158void *krealloc(const void *p, size_t new_size, gfp_t flags)
159{
160 void *ret;
Christoph Lameteref2ad802007-07-17 04:03:21 -0700161
162 if (unlikely(!new_size)) {
163 kfree(p);
Christoph Lameter6cb8f912007-07-17 04:03:22 -0700164 return ZERO_SIZE_PTR;
Christoph Lameteref2ad802007-07-17 04:03:21 -0700165 }
166
Ezequiel Garciae21827a2012-08-14 09:55:21 -0300167 ret = __do_krealloc(p, new_size, flags);
Pekka Enberg93bc4e82008-07-26 17:49:33 -0700168 if (ret && p != ret)
Christoph Lameteref2ad802007-07-17 04:03:21 -0700169 kfree(p);
Pekka Enberg93bc4e82008-07-26 17:49:33 -0700170
Christoph Lameteref2ad802007-07-17 04:03:21 -0700171 return ret;
172}
173EXPORT_SYMBOL(krealloc);
174
Johannes Weiner3ef0e5b2009-02-20 15:38:41 -0800175/**
176 * kzfree - like kfree but zero memory
177 * @p: object to free memory of
178 *
179 * The memory of the object @p points to is zeroed before freed.
180 * If @p is %NULL, kzfree() does nothing.
Pekka Enberga234bdc2009-05-31 13:50:38 +0300181 *
182 * Note: this function zeroes the whole allocated buffer which can be a good
183 * deal bigger than the requested buffer size passed to kmalloc(). So be
184 * careful when using this function in performance sensitive code.
Johannes Weiner3ef0e5b2009-02-20 15:38:41 -0800185 */
186void kzfree(const void *p)
187{
188 size_t ks;
189 void *mem = (void *)p;
190
191 if (unlikely(ZERO_OR_NULL_PTR(mem)))
192 return;
193 ks = ksize(mem);
194 memset(mem, 0, ks);
195 kfree(mem);
196}
197EXPORT_SYMBOL(kzfree);
198
Davi Arnaut96840aa2006-03-24 03:18:42 -0800199/*
200 * strndup_user - duplicate an existing string from user space
Davi Arnaut96840aa2006-03-24 03:18:42 -0800201 * @s: The string to duplicate
202 * @n: Maximum number of bytes to copy, including the trailing NUL.
203 */
204char *strndup_user(const char __user *s, long n)
205{
206 char *p;
207 long length;
208
209 length = strnlen_user(s, n);
210
211 if (!length)
212 return ERR_PTR(-EFAULT);
213
214 if (length > n)
215 return ERR_PTR(-EINVAL);
216
Julia Lawall90d74042010-08-09 17:18:26 -0700217 p = memdup_user(s, length);
Davi Arnaut96840aa2006-03-24 03:18:42 -0800218
Julia Lawall90d74042010-08-09 17:18:26 -0700219 if (IS_ERR(p))
220 return p;
Davi Arnaut96840aa2006-03-24 03:18:42 -0800221
222 p[length - 1] = '\0';
223
224 return p;
225}
226EXPORT_SYMBOL(strndup_user);
Andrew Morton16d69262008-07-25 19:44:36 -0700227
Namhyung Kim6038def2011-05-24 17:11:22 -0700228void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
229 struct vm_area_struct *prev, struct rb_node *rb_parent)
230{
231 struct vm_area_struct *next;
232
233 vma->vm_prev = prev;
234 if (prev) {
235 next = prev->vm_next;
236 prev->vm_next = vma;
237 } else {
238 mm->mmap = vma;
239 if (rb_parent)
240 next = rb_entry(rb_parent,
241 struct vm_area_struct, vm_rb);
242 else
243 next = NULL;
244 }
245 vma->vm_next = next;
246 if (next)
247 next->vm_prev = vma;
248}
249
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700250/* Check if the vma is being used as a stack by this task */
251static int vm_is_stack_for_task(struct task_struct *t,
252 struct vm_area_struct *vma)
253{
254 return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
255}
256
257/*
258 * Check if the vma is being used as a stack.
259 * If is_group is non-zero, check in the entire thread group or else
260 * just check in the current task. Returns the pid of the task that
261 * the vma is stack for.
262 */
263pid_t vm_is_stack(struct task_struct *task,
264 struct vm_area_struct *vma, int in_group)
265{
266 pid_t ret = 0;
267
268 if (vm_is_stack_for_task(task, vma))
269 return task->pid;
270
271 if (in_group) {
272 struct task_struct *t;
273 rcu_read_lock();
274 if (!pid_alive(task))
275 goto done;
276
277 t = task;
278 do {
279 if (vm_is_stack_for_task(t, vma)) {
280 ret = t->pid;
281 goto done;
282 }
283 } while_each_thread(task, t);
284done:
285 rcu_read_unlock();
286 }
287
288 return ret;
289}
290
David Howellsefc1a3b2010-01-15 17:01:35 -0800291#if defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
Andrew Morton16d69262008-07-25 19:44:36 -0700292void arch_pick_mmap_layout(struct mm_struct *mm)
293{
294 mm->mmap_base = TASK_UNMAPPED_BASE;
295 mm->get_unmapped_area = arch_get_unmapped_area;
296 mm->unmap_area = arch_unmap_area;
297}
298#endif
Rusty Russell912985d2008-08-12 17:52:52 -0500299
Xiao Guangrong45888a02010-08-22 19:08:57 +0800300/*
301 * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
302 * back to the regular GUP.
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300303 * If the architecture not support this function, simply return with no
Xiao Guangrong45888a02010-08-22 19:08:57 +0800304 * page pinned
305 */
306int __attribute__((weak)) __get_user_pages_fast(unsigned long start,
307 int nr_pages, int write, struct page **pages)
308{
309 return 0;
310}
311EXPORT_SYMBOL_GPL(__get_user_pages_fast);
312
Andy Grover9de100d2009-04-13 14:40:05 -0700313/**
314 * get_user_pages_fast() - pin user pages in memory
315 * @start: starting user address
316 * @nr_pages: number of pages from start to pin
317 * @write: whether pages will be written to
318 * @pages: array that receives pointers to the pages pinned.
319 * Should be at least nr_pages long.
320 *
Andy Grover9de100d2009-04-13 14:40:05 -0700321 * Returns number of pages pinned. This may be fewer than the number
322 * requested. If nr_pages is 0 or negative, returns 0. If no pages
323 * were pinned, returns -errno.
Nick Piggind2bf6be2009-06-16 15:31:39 -0700324 *
325 * get_user_pages_fast provides equivalent functionality to get_user_pages,
326 * operating on current and current->mm, with force=0 and vma=NULL. However
327 * unlike get_user_pages, it must be called without mmap_sem held.
328 *
329 * get_user_pages_fast may take mmap_sem and page table locks, so no
330 * assumptions can be made about lack of locking. get_user_pages_fast is to be
331 * implemented in a way that is advantageous (vs get_user_pages()) when the
332 * user memory area is already faulted in and present in ptes. However if the
333 * pages have to be faulted in, it may turn out to be slightly slower so
334 * callers need to carefully consider what to use. On many architectures,
335 * get_user_pages_fast simply falls back to get_user_pages.
Andy Grover9de100d2009-04-13 14:40:05 -0700336 */
Rusty Russell912985d2008-08-12 17:52:52 -0500337int __attribute__((weak)) get_user_pages_fast(unsigned long start,
338 int nr_pages, int write, struct page **pages)
339{
340 struct mm_struct *mm = current->mm;
341 int ret;
342
343 down_read(&mm->mmap_sem);
344 ret = get_user_pages(current, mm, start, nr_pages,
345 write, 0, pages, NULL);
346 up_read(&mm->mmap_sem);
347
348 return ret;
349}
350EXPORT_SYMBOL_GPL(get_user_pages_fast);
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +0200351
Al Viroeb36c582012-05-30 20:17:35 -0400352unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
353 unsigned long len, unsigned long prot,
354 unsigned long flag, unsigned long pgoff)
355{
356 unsigned long ret;
357 struct mm_struct *mm = current->mm;
358
359 ret = security_mmap_file(file, prot, flag);
360 if (!ret) {
361 down_write(&mm->mmap_sem);
362 ret = do_mmap_pgoff(file, addr, len, prot, flag, pgoff);
363 up_write(&mm->mmap_sem);
364 }
365 return ret;
366}
367
368unsigned long vm_mmap(struct file *file, unsigned long addr,
369 unsigned long len, unsigned long prot,
370 unsigned long flag, unsigned long offset)
371{
372 if (unlikely(offset + PAGE_ALIGN(len) < offset))
373 return -EINVAL;
374 if (unlikely(offset & ~PAGE_MASK))
375 return -EINVAL;
376
377 return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
378}
379EXPORT_SYMBOL(vm_mmap);
380
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +0200381/* Tracepoints definitions. */
Eduard - Gabriel Munteanuca2b84cb2009-03-23 15:12:24 +0200382EXPORT_TRACEPOINT_SYMBOL(kmalloc);
383EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
384EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
385EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
386EXPORT_TRACEPOINT_SYMBOL(kfree);
387EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);