blob: fa95ab2d36740803e06da898a4851db317b86a9e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#include <linux/mm.h>
Davidlohr Bueso615d6e82014-04-07 15:37:25 -07002#include <linux/vmacache.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07003#include <linux/hugetlb.h>
Dave Hansen22e057c2011-03-22 16:33:00 -07004#include <linux/huge_mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/mount.h>
6#include <linux/seq_file.h>
Mauricio Line070ad42005-09-03 15:55:10 -07007#include <linux/highmem.h>
Kees Cook5096add2007-05-08 00:26:04 -07008#include <linux/ptrace.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09009#include <linux/slab.h>
Christoph Lameter6e21c8f2005-09-03 15:54:45 -070010#include <linux/pagemap.h>
11#include <linux/mempolicy.h>
Dave Hansen22e057c2011-03-22 16:33:00 -070012#include <linux/rmap.h>
Matt Mackall85863e42008-02-04 22:29:04 -080013#include <linux/swap.h>
14#include <linux/swapops.h>
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -070015#include <linux/mmu_notifier.h>
Vladimir Davydov33c3fc72015-09-09 15:35:45 -070016#include <linux/page_idle.h>
Vlastimil Babka6a15a372016-01-14 15:19:20 -080017#include <linux/shmem_fs.h>
Mauricio Line070ad42005-09-03 15:55:10 -070018
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <asm/elf.h>
20#include <asm/uaccess.h>
Mauricio Line070ad42005-09-03 15:55:10 -070021#include <asm/tlbflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include "internal.h"
23
Eric W. Biedermandf5f8312008-02-08 04:18:33 -080024void task_mem(struct seq_file *m, struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -070025{
Konstantin Khlebnikov84638332016-01-14 15:22:07 -080026 unsigned long text, lib, swap, ptes, pmds, anon, file, shmem;
Hugh Dickins365e9c872005-10-29 18:16:18 -070027 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
28
Jerome Marchand8cee8522016-01-14 15:19:29 -080029 anon = get_mm_counter(mm, MM_ANONPAGES);
30 file = get_mm_counter(mm, MM_FILEPAGES);
31 shmem = get_mm_counter(mm, MM_SHMEMPAGES);
32
Hugh Dickins365e9c872005-10-29 18:16:18 -070033 /*
34 * Note: to minimize their overhead, mm maintains hiwater_vm and
35 * hiwater_rss only when about to *lower* total_vm or rss. Any
36 * collector of these hiwater stats must therefore get total_vm
37 * and rss too, which will usually be the higher. Barriers? not
38 * worth the effort, such snapshots can always be inconsistent.
39 */
40 hiwater_vm = total_vm = mm->total_vm;
41 if (hiwater_vm < mm->hiwater_vm)
42 hiwater_vm = mm->hiwater_vm;
Jerome Marchand8cee8522016-01-14 15:19:29 -080043 hiwater_rss = total_rss = anon + file + shmem;
Hugh Dickins365e9c872005-10-29 18:16:18 -070044 if (hiwater_rss < mm->hiwater_rss)
45 hiwater_rss = mm->hiwater_rss;
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
48 lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080049 swap = get_mm_counter(mm, MM_SWAPENTS);
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -080050 ptes = PTRS_PER_PTE * sizeof(pte_t) * atomic_long_read(&mm->nr_ptes);
51 pmds = PTRS_PER_PMD * sizeof(pmd_t) * mm_nr_pmds(mm);
Eric W. Biedermandf5f8312008-02-08 04:18:33 -080052 seq_printf(m,
Hugh Dickins365e9c872005-10-29 18:16:18 -070053 "VmPeak:\t%8lu kB\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 "VmSize:\t%8lu kB\n"
55 "VmLck:\t%8lu kB\n"
Christoph Lameterbc3e53f2011-10-31 17:07:30 -070056 "VmPin:\t%8lu kB\n"
Hugh Dickins365e9c872005-10-29 18:16:18 -070057 "VmHWM:\t%8lu kB\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 "VmRSS:\t%8lu kB\n"
Jerome Marchand8cee8522016-01-14 15:19:29 -080059 "RssAnon:\t%8lu kB\n"
60 "RssFile:\t%8lu kB\n"
61 "RssShmem:\t%8lu kB\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 "VmData:\t%8lu kB\n"
63 "VmStk:\t%8lu kB\n"
64 "VmExe:\t%8lu kB\n"
65 "VmLib:\t%8lu kB\n"
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080066 "VmPTE:\t%8lu kB\n"
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -080067 "VmPMD:\t%8lu kB\n"
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080068 "VmSwap:\t%8lu kB\n",
Hugh Dickins365e9c872005-10-29 18:16:18 -070069 hiwater_vm << (PAGE_SHIFT-10),
Konstantin Khlebnikov314e51b2012-10-08 16:29:02 -070070 total_vm << (PAGE_SHIFT-10),
Linus Torvalds1da177e2005-04-16 15:20:36 -070071 mm->locked_vm << (PAGE_SHIFT-10),
Christoph Lameterbc3e53f2011-10-31 17:07:30 -070072 mm->pinned_vm << (PAGE_SHIFT-10),
Hugh Dickins365e9c872005-10-29 18:16:18 -070073 hiwater_rss << (PAGE_SHIFT-10),
74 total_rss << (PAGE_SHIFT-10),
Jerome Marchand8cee8522016-01-14 15:19:29 -080075 anon << (PAGE_SHIFT-10),
76 file << (PAGE_SHIFT-10),
77 shmem << (PAGE_SHIFT-10),
Konstantin Khlebnikov84638332016-01-14 15:22:07 -080078 mm->data_vm << (PAGE_SHIFT-10),
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 mm->stack_vm << (PAGE_SHIFT-10), text, lib,
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -080080 ptes >> 10,
81 pmds >> 10,
KAMEZAWA Hiroyukib084d432010-03-05 13:41:42 -080082 swap << (PAGE_SHIFT-10));
Naoya Horiguchi5d317b22015-11-05 18:47:14 -080083 hugetlb_report_usage(m, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -070084}
85
86unsigned long task_vsize(struct mm_struct *mm)
87{
88 return PAGE_SIZE * mm->total_vm;
89}
90
Alexey Dobriyana2ade7b2011-01-12 17:00:32 -080091unsigned long task_statm(struct mm_struct *mm,
92 unsigned long *shared, unsigned long *text,
93 unsigned long *data, unsigned long *resident)
Linus Torvalds1da177e2005-04-16 15:20:36 -070094{
Jerome Marchandeca56ff2016-01-14 15:19:26 -080095 *shared = get_mm_counter(mm, MM_FILEPAGES) +
96 get_mm_counter(mm, MM_SHMEMPAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
98 >> PAGE_SHIFT;
Konstantin Khlebnikov84638332016-01-14 15:22:07 -080099 *data = mm->data_vm + mm->stack_vm;
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -0800100 *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 return mm->total_vm;
102}
103
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900104#ifdef CONFIG_NUMA
105/*
Oleg Nesterov498f2372014-10-09 15:27:52 -0700106 * Save get_task_policy() for show_numa_map().
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900107 */
108static void hold_task_mempolicy(struct proc_maps_private *priv)
109{
110 struct task_struct *task = priv->task;
111
112 task_lock(task);
Oleg Nesterov498f2372014-10-09 15:27:52 -0700113 priv->task_mempolicy = get_task_policy(task);
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900114 mpol_get(priv->task_mempolicy);
115 task_unlock(task);
116}
117static void release_task_mempolicy(struct proc_maps_private *priv)
118{
119 mpol_put(priv->task_mempolicy);
120}
121#else
122static void hold_task_mempolicy(struct proc_maps_private *priv)
123{
124}
125static void release_task_mempolicy(struct proc_maps_private *priv)
126{
127}
128#endif
129
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700130static void vma_stop(struct proc_maps_private *priv)
Mauricio Line070ad42005-09-03 15:55:10 -0700131{
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700132 struct mm_struct *mm = priv->mm;
133
134 release_task_mempolicy(priv);
135 up_read(&mm->mmap_sem);
136 mmput(mm);
Matt Mackalla6198792008-02-04 22:29:03 -0800137}
138
Oleg Nesterovad2a00e2014-10-09 15:25:39 -0700139static struct vm_area_struct *
140m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma)
141{
142 if (vma == priv->tail_vma)
143 return NULL;
144 return vma->vm_next ?: priv->tail_vma;
145}
146
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700147static void m_cache_vma(struct seq_file *m, struct vm_area_struct *vma)
148{
149 if (m->count < m->size) /* vma is copied successfully */
150 m->version = m_next_vma(m->private, vma) ? vma->vm_start : -1UL;
151}
152
Oleg Nesterov0c255322014-10-09 15:25:36 -0700153static void *m_start(struct seq_file *m, loff_t *ppos)
Matt Mackalla6198792008-02-04 22:29:03 -0800154{
155 struct proc_maps_private *priv = m->private;
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700156 unsigned long last_addr = m->version;
Matt Mackalla6198792008-02-04 22:29:03 -0800157 struct mm_struct *mm;
Oleg Nesterov0c255322014-10-09 15:25:36 -0700158 struct vm_area_struct *vma;
159 unsigned int pos = *ppos;
Matt Mackalla6198792008-02-04 22:29:03 -0800160
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700161 /* See m_cache_vma(). Zero at the start or after lseek. */
162 if (last_addr == -1UL)
163 return NULL;
164
Oleg Nesterov2c033762014-10-09 15:25:51 -0700165 priv->task = get_proc_task(priv->inode);
Matt Mackalla6198792008-02-04 22:29:03 -0800166 if (!priv->task)
Al Viroec6fd8a2011-02-15 22:22:54 -0500167 return ERR_PTR(-ESRCH);
Matt Mackalla6198792008-02-04 22:29:03 -0800168
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700169 mm = priv->mm;
170 if (!mm || !atomic_inc_not_zero(&mm->mm_users))
171 return NULL;
Oleg Nesterov0c255322014-10-09 15:25:36 -0700172
Oleg Nesterov00f89d22009-07-10 03:27:38 +0200173 down_read(&mm->mmap_sem);
KAMEZAWA Hiroyuki9e781442012-10-19 17:00:55 +0900174 hold_task_mempolicy(priv);
Oleg Nesterov0c255322014-10-09 15:25:36 -0700175 priv->tail_vma = get_gate_vma(mm);
Matt Mackalla6198792008-02-04 22:29:03 -0800176
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700177 if (last_addr) {
178 vma = find_vma(mm, last_addr);
179 if (vma && (vma = m_next_vma(priv, vma)))
180 return vma;
181 }
182
183 m->version = 0;
Oleg Nesterov0c255322014-10-09 15:25:36 -0700184 if (pos < mm->map_count) {
Oleg Nesterov557c2d82014-10-09 15:25:43 -0700185 for (vma = mm->mmap; pos; pos--) {
186 m->version = vma->vm_start;
Matt Mackalla6198792008-02-04 22:29:03 -0800187 vma = vma->vm_next;
Oleg Nesterov557c2d82014-10-09 15:25:43 -0700188 }
Oleg Nesterov0c255322014-10-09 15:25:36 -0700189 return vma;
Matt Mackalla6198792008-02-04 22:29:03 -0800190 }
191
Oleg Nesterov557c2d82014-10-09 15:25:43 -0700192 /* we do not bother to update m->version in this case */
Oleg Nesterov0c255322014-10-09 15:25:36 -0700193 if (pos == mm->map_count && priv->tail_vma)
194 return priv->tail_vma;
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700195
196 vma_stop(priv);
197 return NULL;
Matt Mackalla6198792008-02-04 22:29:03 -0800198}
199
200static void *m_next(struct seq_file *m, void *v, loff_t *pos)
201{
202 struct proc_maps_private *priv = m->private;
Oleg Nesterovad2a00e2014-10-09 15:25:39 -0700203 struct vm_area_struct *next;
Matt Mackalla6198792008-02-04 22:29:03 -0800204
205 (*pos)++;
Oleg Nesterovad2a00e2014-10-09 15:25:39 -0700206 next = m_next_vma(priv, v);
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700207 if (!next)
208 vma_stop(priv);
209 return next;
Matt Mackalla6198792008-02-04 22:29:03 -0800210}
211
212static void m_stop(struct seq_file *m, void *v)
213{
214 struct proc_maps_private *priv = m->private;
Matt Mackalla6198792008-02-04 22:29:03 -0800215
Oleg Nesterov59b4bf12014-10-09 15:25:28 -0700216 if (!IS_ERR_OR_NULL(v))
217 vma_stop(priv);
Oleg Nesterov0d5f5f42014-10-09 15:25:32 -0700218 if (priv->task) {
Matt Mackalla6198792008-02-04 22:29:03 -0800219 put_task_struct(priv->task);
Oleg Nesterov0d5f5f42014-10-09 15:25:32 -0700220 priv->task = NULL;
221 }
Matt Mackalla6198792008-02-04 22:29:03 -0800222}
223
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -0700224static int proc_maps_open(struct inode *inode, struct file *file,
225 const struct seq_operations *ops, int psize)
226{
227 struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
228
229 if (!priv)
230 return -ENOMEM;
231
Oleg Nesterov2c033762014-10-09 15:25:51 -0700232 priv->inode = inode;
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700233 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
234 if (IS_ERR(priv->mm)) {
235 int err = PTR_ERR(priv->mm);
236
237 seq_release_private(inode, file);
238 return err;
239 }
240
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -0700241 return 0;
242}
243
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700244static int proc_map_release(struct inode *inode, struct file *file)
245{
246 struct seq_file *seq = file->private_data;
247 struct proc_maps_private *priv = seq->private;
248
249 if (priv->mm)
250 mmdrop(priv->mm);
251
252 return seq_release_private(inode, file);
253}
254
Matt Mackalla6198792008-02-04 22:29:03 -0800255static int do_maps_open(struct inode *inode, struct file *file,
Jan Engelhardt03a44822008-02-08 04:21:19 -0800256 const struct seq_operations *ops)
Matt Mackalla6198792008-02-04 22:29:03 -0800257{
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -0700258 return proc_maps_open(inode, file, ops,
259 sizeof(struct proc_maps_private));
Matt Mackalla6198792008-02-04 22:29:03 -0800260}
Mauricio Line070ad42005-09-03 15:55:10 -0700261
Johannes Weiner65376df2016-02-02 16:57:29 -0800262/*
263 * Indicate if the VMA is a stack for the given task; for
264 * /proc/PID/maps that is the stack of the main task.
265 */
266static int is_stack(struct proc_maps_private *priv,
267 struct vm_area_struct *vma, int is_pid)
Oleg Nesterov58cb6542014-10-09 15:25:54 -0700268{
Johannes Weiner65376df2016-02-02 16:57:29 -0800269 int stack = 0;
Oleg Nesterov58cb6542014-10-09 15:25:54 -0700270
Johannes Weiner65376df2016-02-02 16:57:29 -0800271 if (is_pid) {
272 stack = vma->vm_start <= vma->vm_mm->start_stack &&
273 vma->vm_end >= vma->vm_mm->start_stack;
274 } else {
275 struct inode *inode = priv->inode;
276 struct task_struct *task;
277
278 rcu_read_lock();
279 task = pid_task(proc_pid(inode), PIDTYPE_PID);
Oleg Nesterov58cb6542014-10-09 15:25:54 -0700280 if (task)
Johannes Weiner65376df2016-02-02 16:57:29 -0800281 stack = vma_is_stack_for_task(vma, task);
282 rcu_read_unlock();
Oleg Nesterov58cb6542014-10-09 15:25:54 -0700283 }
Johannes Weiner65376df2016-02-02 16:57:29 -0800284 return stack;
Oleg Nesterov58cb6542014-10-09 15:25:54 -0700285}
286
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700287static void
288show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289{
Mauricio Line070ad42005-09-03 15:55:10 -0700290 struct mm_struct *mm = vma->vm_mm;
291 struct file *file = vma->vm_file;
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700292 struct proc_maps_private *priv = m->private;
KOSAKI Motohiroca16d142011-05-26 19:16:19 +0900293 vm_flags_t flags = vma->vm_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 unsigned long ino = 0;
KAMEZAWA Hiroyuki6260a4b2009-04-06 19:00:30 -0700295 unsigned long long pgoff = 0;
Mikulas Patockaa09a79f2011-05-09 13:01:09 +0200296 unsigned long start, end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 dev_t dev = 0;
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700298 const char *name = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
300 if (file) {
Al Viro496ad9a2013-01-23 17:07:38 -0500301 struct inode *inode = file_inode(vma->vm_file);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 dev = inode->i_sb->s_dev;
303 ino = inode->i_ino;
KAMEZAWA Hiroyuki6260a4b2009-04-06 19:00:30 -0700304 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 }
306
Linus Torvaldsd7824372010-08-15 11:35:52 -0700307 /* We don't show the stack guard page in /proc/maps */
308 start = vma->vm_start;
Mikulas Patockaa09a79f2011-05-09 13:01:09 +0200309 if (stack_guard_page_start(vma, start))
310 start += PAGE_SIZE;
311 end = vma->vm_end;
312 if (stack_guard_page_end(vma, end))
313 end -= PAGE_SIZE;
Linus Torvaldsd7824372010-08-15 11:35:52 -0700314
Tetsuo Handa652586d2013-11-14 14:31:57 -0800315 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
316 seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu ",
Linus Torvaldsd7824372010-08-15 11:35:52 -0700317 start,
Mikulas Patockaa09a79f2011-05-09 13:01:09 +0200318 end,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 flags & VM_READ ? 'r' : '-',
320 flags & VM_WRITE ? 'w' : '-',
321 flags & VM_EXEC ? 'x' : '-',
322 flags & VM_MAYSHARE ? 's' : 'p',
KAMEZAWA Hiroyuki6260a4b2009-04-06 19:00:30 -0700323 pgoff,
Tetsuo Handa652586d2013-11-14 14:31:57 -0800324 MAJOR(dev), MINOR(dev), ino);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325
326 /*
327 * Print the dentry name for named mappings, and a
328 * special [heap] marker for the heap:
329 */
Mauricio Line070ad42005-09-03 15:55:10 -0700330 if (file) {
Tetsuo Handa652586d2013-11-14 14:31:57 -0800331 seq_pad(m, ' ');
Miklos Szeredi2726d562015-06-19 10:30:28 +0200332 seq_file_path(m, file, "\n");
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700333 goto done;
334 }
335
Andy Lutomirski78d683e2014-05-19 15:58:32 -0700336 if (vma->vm_ops && vma->vm_ops->name) {
337 name = vma->vm_ops->name(vma);
338 if (name)
339 goto done;
340 }
341
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700342 name = arch_vma_name(vma);
343 if (!name) {
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700344 if (!mm) {
345 name = "[vdso]";
346 goto done;
347 }
348
349 if (vma->vm_start <= mm->brk &&
350 vma->vm_end >= mm->start_brk) {
351 name = "[heap]";
352 goto done;
353 }
354
Johannes Weiner65376df2016-02-02 16:57:29 -0800355 if (is_stack(priv, vma, is_pid))
356 name = "[stack]";
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700357 }
358
359done:
360 if (name) {
Tetsuo Handa652586d2013-11-14 14:31:57 -0800361 seq_pad(m, ' ');
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700362 seq_puts(m, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 }
364 seq_putc(m, '\n');
Joe Korty7c88db02008-10-16 15:27:09 +0400365}
366
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700367static int show_map(struct seq_file *m, void *v, int is_pid)
Joe Korty7c88db02008-10-16 15:27:09 +0400368{
Oleg Nesterovebb6cdd2014-10-09 15:25:34 -0700369 show_map_vma(m, v, is_pid);
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700370 m_cache_vma(m, v);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 return 0;
372}
373
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700374static int show_pid_map(struct seq_file *m, void *v)
375{
376 return show_map(m, v, 1);
377}
378
379static int show_tid_map(struct seq_file *m, void *v)
380{
381 return show_map(m, v, 0);
382}
383
Jan Engelhardt03a44822008-02-08 04:21:19 -0800384static const struct seq_operations proc_pid_maps_op = {
Matt Mackalla6198792008-02-04 22:29:03 -0800385 .start = m_start,
386 .next = m_next,
387 .stop = m_stop,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700388 .show = show_pid_map
Matt Mackalla6198792008-02-04 22:29:03 -0800389};
390
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700391static const struct seq_operations proc_tid_maps_op = {
392 .start = m_start,
393 .next = m_next,
394 .stop = m_stop,
395 .show = show_tid_map
396};
397
398static int pid_maps_open(struct inode *inode, struct file *file)
Matt Mackalla6198792008-02-04 22:29:03 -0800399{
400 return do_maps_open(inode, file, &proc_pid_maps_op);
401}
402
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700403static int tid_maps_open(struct inode *inode, struct file *file)
404{
405 return do_maps_open(inode, file, &proc_tid_maps_op);
406}
407
408const struct file_operations proc_pid_maps_operations = {
409 .open = pid_maps_open,
410 .read = seq_read,
411 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700412 .release = proc_map_release,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700413};
414
415const struct file_operations proc_tid_maps_operations = {
416 .open = tid_maps_open,
Matt Mackalla6198792008-02-04 22:29:03 -0800417 .read = seq_read,
418 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700419 .release = proc_map_release,
Matt Mackalla6198792008-02-04 22:29:03 -0800420};
421
422/*
423 * Proportional Set Size(PSS): my share of RSS.
424 *
425 * PSS of a process is the count of pages it has in memory, where each
426 * page is divided by the number of processes sharing it. So if a
427 * process has 1000 pages all to itself, and 1000 shared with one other
428 * process, its PSS will be 1500.
429 *
430 * To keep (accumulated) division errors low, we adopt a 64bit
431 * fixed-point pss counter to minimize division errors. So (pss >>
432 * PSS_SHIFT) would be the real byte count.
433 *
434 * A shift of 12 before division means (assuming 4K page size):
435 * - 1M 3-user-pages add up to 8KB errors;
436 * - supports mapcount up to 2^24, or 16M;
437 * - supports PSS up to 2^52 bytes, or 4PB.
438 */
439#define PSS_SHIFT 12
440
Matt Mackall1e883282008-02-04 22:29:07 -0800441#ifdef CONFIG_PROC_PAGE_MONITOR
Peter Zijlstra214e4712008-04-28 02:12:55 -0700442struct mem_size_stats {
Matt Mackalla6198792008-02-04 22:29:03 -0800443 unsigned long resident;
444 unsigned long shared_clean;
445 unsigned long shared_dirty;
446 unsigned long private_clean;
447 unsigned long private_dirty;
448 unsigned long referenced;
Nikanth Karthikesanb40d4f82010-10-27 15:34:10 -0700449 unsigned long anonymous;
Dave Hansen4031a212011-03-22 16:33:01 -0700450 unsigned long anonymous_thp;
Peter Zijlstra214e4712008-04-28 02:12:55 -0700451 unsigned long swap;
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800452 unsigned long shared_hugetlb;
453 unsigned long private_hugetlb;
Matt Mackalla6198792008-02-04 22:29:03 -0800454 u64 pss;
Minchan Kim8334b962015-09-08 15:00:24 -0700455 u64 swap_pss;
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800456 bool check_shmem_swap;
Matt Mackalla6198792008-02-04 22:29:03 -0800457};
458
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800459static void smaps_account(struct mem_size_stats *mss, struct page *page,
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800460 bool compound, bool young, bool dirty)
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800461{
Kirill A. Shutemovf4be6152016-01-20 14:58:12 -0800462 int i, nr = compound ? 1 << compound_order(page) : 1;
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800463 unsigned long size = nr * PAGE_SIZE;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700464
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800465 if (PageAnon(page))
466 mss->anonymous += size;
467
468 mss->resident += size;
469 /* Accumulate the size in pages that have been accessed. */
Vladimir Davydov33c3fc72015-09-09 15:35:45 -0700470 if (young || page_is_young(page) || PageReferenced(page))
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800471 mss->referenced += size;
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800472
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800473 /*
474 * page_count(page) == 1 guarantees the page is mapped exactly once.
475 * If any subpage of the compound page mapped with PTE it would elevate
476 * page_count().
477 */
478 if (page_count(page) == 1) {
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800479 if (dirty || PageDirty(page))
480 mss->private_dirty += size;
481 else
482 mss->private_clean += size;
483 mss->pss += (u64)size << PSS_SHIFT;
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800484 return;
485 }
486
487 for (i = 0; i < nr; i++, page++) {
488 int mapcount = page_mapcount(page);
489
490 if (mapcount >= 2) {
491 if (dirty || PageDirty(page))
492 mss->shared_dirty += PAGE_SIZE;
493 else
494 mss->shared_clean += PAGE_SIZE;
495 mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
496 } else {
497 if (dirty || PageDirty(page))
498 mss->private_dirty += PAGE_SIZE;
499 else
500 mss->private_clean += PAGE_SIZE;
501 mss->pss += PAGE_SIZE << PSS_SHIFT;
502 }
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800503 }
504}
505
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800506#ifdef CONFIG_SHMEM
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800507static int smaps_pte_hole(unsigned long addr, unsigned long end,
508 struct mm_walk *walk)
509{
510 struct mem_size_stats *mss = walk->private;
511
Vlastimil Babka48131e02016-01-14 15:19:23 -0800512 mss->swap += shmem_partial_swap_usage(
513 walk->vma->vm_file->f_mapping, addr, end);
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800514
515 return 0;
516}
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800517#endif
518
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800519static void smaps_pte_entry(pte_t *pte, unsigned long addr,
520 struct mm_walk *walk)
Dave Hansenae11c4d2011-03-22 16:32:58 -0700521{
522 struct mem_size_stats *mss = walk->private;
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800523 struct vm_area_struct *vma = walk->vma;
Konstantin Khlebnikovb1d4d9e2012-05-31 16:26:20 -0700524 struct page *page = NULL;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700525
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800526 if (pte_present(*pte)) {
527 page = vm_normal_page(vma, addr, *pte);
528 } else if (is_swap_pte(*pte)) {
529 swp_entry_t swpent = pte_to_swp_entry(*pte);
Konstantin Khlebnikovb1d4d9e2012-05-31 16:26:20 -0700530
Minchan Kim8334b962015-09-08 15:00:24 -0700531 if (!non_swap_entry(swpent)) {
532 int mapcount;
533
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800534 mss->swap += PAGE_SIZE;
Minchan Kim8334b962015-09-08 15:00:24 -0700535 mapcount = swp_swapcount(swpent);
536 if (mapcount >= 2) {
537 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
538
539 do_div(pss_delta, mapcount);
540 mss->swap_pss += pss_delta;
541 } else {
542 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
543 }
544 } else if (is_migration_entry(swpent))
Konstantin Khlebnikovb1d4d9e2012-05-31 16:26:20 -0700545 page = migration_entry_to_page(swpent);
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800546 } else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
547 && pte_none(*pte))) {
Vlastimil Babka48131e02016-01-14 15:19:23 -0800548 page = find_get_entry(vma->vm_file->f_mapping,
549 linear_page_index(vma, addr));
550 if (!page)
551 return;
552
553 if (radix_tree_exceptional_entry(page))
554 mss->swap += PAGE_SIZE;
555 else
556 page_cache_release(page);
557
558 return;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700559 }
560
Dave Hansenae11c4d2011-03-22 16:32:58 -0700561 if (!page)
562 return;
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800563
564 smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte));
Dave Hansenae11c4d2011-03-22 16:32:58 -0700565}
566
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800567#ifdef CONFIG_TRANSPARENT_HUGEPAGE
568static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
569 struct mm_walk *walk)
570{
571 struct mem_size_stats *mss = walk->private;
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800572 struct vm_area_struct *vma = walk->vma;
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800573 struct page *page;
574
575 /* FOLL_DUMP will return -EFAULT on huge zero page */
576 page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
577 if (IS_ERR_OR_NULL(page))
578 return;
579 mss->anonymous_thp += HPAGE_PMD_SIZE;
Kirill A. Shutemovafd98832016-01-15 16:52:13 -0800580 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd));
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800581}
582#else
583static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
584 struct mm_walk *walk)
585{
586}
587#endif
588
Matt Mackallb3ae5ac2008-02-04 22:29:01 -0800589static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -0700590 struct mm_walk *walk)
Mauricio Line070ad42005-09-03 15:55:10 -0700591{
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800592 struct vm_area_struct *vma = walk->vma;
Dave Hansenae11c4d2011-03-22 16:32:58 -0700593 pte_t *pte;
Hugh Dickins705e87c2005-10-29 18:16:27 -0700594 spinlock_t *ptl;
Mauricio Line070ad42005-09-03 15:55:10 -0700595
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -0800596 ptl = pmd_trans_huge_lock(pmd, vma);
597 if (ptl) {
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800598 smaps_pmd_entry(pmd, addr, walk);
Kirill A. Shutemovbf929152013-11-14 14:30:54 -0800599 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -0700600 return 0;
Dave Hansen22e057c2011-03-22 16:33:00 -0700601 }
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -0700602
603 if (pmd_trans_unstable(pmd))
604 return 0;
Dave Hansen22e057c2011-03-22 16:33:00 -0700605 /*
606 * The mmap_sem held all the way back in m_start() is what
607 * keeps khugepaged out of here and from collapsing things
608 * in here.
609 */
Hugh Dickins705e87c2005-10-29 18:16:27 -0700610 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
Dave Hansenae11c4d2011-03-22 16:32:58 -0700611 for (; addr != end; pte++, addr += PAGE_SIZE)
Kirill A. Shutemovc164e032014-12-10 15:44:36 -0800612 smaps_pte_entry(pte, addr, walk);
Hugh Dickins705e87c2005-10-29 18:16:27 -0700613 pte_unmap_unlock(pte - 1, ptl);
614 cond_resched();
Matt Mackallb3ae5ac2008-02-04 22:29:01 -0800615 return 0;
Mauricio Line070ad42005-09-03 15:55:10 -0700616}
617
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800618static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
619{
620 /*
621 * Don't forget to update Documentation/ on changes.
622 */
623 static const char mnemonics[BITS_PER_LONG][2] = {
624 /*
625 * In case if we meet a flag we don't know about.
626 */
627 [0 ... (BITS_PER_LONG-1)] = "??",
628
629 [ilog2(VM_READ)] = "rd",
630 [ilog2(VM_WRITE)] = "wr",
631 [ilog2(VM_EXEC)] = "ex",
632 [ilog2(VM_SHARED)] = "sh",
633 [ilog2(VM_MAYREAD)] = "mr",
634 [ilog2(VM_MAYWRITE)] = "mw",
635 [ilog2(VM_MAYEXEC)] = "me",
636 [ilog2(VM_MAYSHARE)] = "ms",
637 [ilog2(VM_GROWSDOWN)] = "gd",
638 [ilog2(VM_PFNMAP)] = "pf",
639 [ilog2(VM_DENYWRITE)] = "dw",
Qiaowei Ren4aae7e42014-11-14 07:18:25 -0800640#ifdef CONFIG_X86_INTEL_MPX
641 [ilog2(VM_MPX)] = "mp",
642#endif
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800643 [ilog2(VM_LOCKED)] = "lo",
644 [ilog2(VM_IO)] = "io",
645 [ilog2(VM_SEQ_READ)] = "sr",
646 [ilog2(VM_RAND_READ)] = "rr",
647 [ilog2(VM_DONTCOPY)] = "dc",
648 [ilog2(VM_DONTEXPAND)] = "de",
649 [ilog2(VM_ACCOUNT)] = "ac",
650 [ilog2(VM_NORESERVE)] = "nr",
651 [ilog2(VM_HUGETLB)] = "ht",
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800652 [ilog2(VM_ARCH_1)] = "ar",
653 [ilog2(VM_DONTDUMP)] = "dd",
Naoya Horiguchiec8e41a2013-11-12 15:07:49 -0800654#ifdef CONFIG_MEM_SOFT_DIRTY
655 [ilog2(VM_SOFTDIRTY)] = "sd",
656#endif
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800657 [ilog2(VM_MIXEDMAP)] = "mm",
658 [ilog2(VM_HUGEPAGE)] = "hg",
659 [ilog2(VM_NOHUGEPAGE)] = "nh",
660 [ilog2(VM_MERGEABLE)] = "mg",
Andrea Arcangeli16ba6f82015-09-04 15:46:17 -0700661 [ilog2(VM_UFFD_MISSING)]= "um",
662 [ilog2(VM_UFFD_WP)] = "uw",
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800663 };
664 size_t i;
665
666 seq_puts(m, "VmFlags: ");
667 for (i = 0; i < BITS_PER_LONG; i++) {
668 if (vma->vm_flags & (1UL << i)) {
669 seq_printf(m, "%c%c ",
670 mnemonics[i][0], mnemonics[i][1]);
671 }
672 }
673 seq_putc(m, '\n');
674}
675
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800676#ifdef CONFIG_HUGETLB_PAGE
677static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
678 unsigned long addr, unsigned long end,
679 struct mm_walk *walk)
680{
681 struct mem_size_stats *mss = walk->private;
682 struct vm_area_struct *vma = walk->vma;
683 struct page *page = NULL;
684
685 if (pte_present(*pte)) {
686 page = vm_normal_page(vma, addr, *pte);
687 } else if (is_swap_pte(*pte)) {
688 swp_entry_t swpent = pte_to_swp_entry(*pte);
689
690 if (is_migration_entry(swpent))
691 page = migration_entry_to_page(swpent);
692 }
693 if (page) {
694 int mapcount = page_mapcount(page);
695
696 if (mapcount >= 2)
697 mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
698 else
699 mss->private_hugetlb += huge_page_size(hstate_vma(vma));
700 }
701 return 0;
702}
703#endif /* HUGETLB_PAGE */
704
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700705static int show_smap(struct seq_file *m, void *v, int is_pid)
Mauricio Line070ad42005-09-03 15:55:10 -0700706{
707 struct vm_area_struct *vma = v;
Mauricio Line070ad42005-09-03 15:55:10 -0700708 struct mem_size_stats mss;
Dave Hansen21650092008-06-12 15:21:47 -0700709 struct mm_walk smaps_walk = {
710 .pmd_entry = smaps_pte_range,
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800711#ifdef CONFIG_HUGETLB_PAGE
712 .hugetlb_entry = smaps_hugetlb_range,
713#endif
Dave Hansen21650092008-06-12 15:21:47 -0700714 .mm = vma->vm_mm,
715 .private = &mss,
716 };
Mauricio Line070ad42005-09-03 15:55:10 -0700717
718 memset(&mss, 0, sizeof mss);
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800719
720#ifdef CONFIG_SHMEM
721 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
Vlastimil Babka6a15a372016-01-14 15:19:20 -0800722 /*
723 * For shared or readonly shmem mappings we know that all
724 * swapped out pages belong to the shmem object, and we can
725 * obtain the swap value much more efficiently. For private
726 * writable mappings, we might have COW pages that are
727 * not affected by the parent swapped out pages of the shmem
728 * object, so we have to distinguish them during the page walk.
729 * Unless we know that the shmem object (or the part mapped by
730 * our VMA) has no swapped out pages at all.
731 */
732 unsigned long shmem_swapped = shmem_swap_usage(vma);
733
734 if (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
735 !(vma->vm_flags & VM_WRITE)) {
736 mss.swap = shmem_swapped;
737 } else {
738 mss.check_shmem_swap = true;
739 smaps_walk.pte_hole = smaps_pte_hole;
740 }
Vlastimil Babkac261e7d2016-01-14 15:19:17 -0800741 }
742#endif
743
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +0900744 /* mmap_sem is held in m_start */
Naoya Horiguchi14eb6fd2015-02-11 15:27:43 -0800745 walk_page_vma(vma, &smaps_walk);
Matt Mackall4752c362008-02-04 22:29:02 -0800746
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700747 show_map_vma(m, vma, is_pid);
Matt Mackall4752c362008-02-04 22:29:02 -0800748
749 seq_printf(m,
750 "Size: %8lu kB\n"
751 "Rss: %8lu kB\n"
752 "Pss: %8lu kB\n"
753 "Shared_Clean: %8lu kB\n"
754 "Shared_Dirty: %8lu kB\n"
755 "Private_Clean: %8lu kB\n"
756 "Private_Dirty: %8lu kB\n"
Peter Zijlstra214e4712008-04-28 02:12:55 -0700757 "Referenced: %8lu kB\n"
Nikanth Karthikesanb40d4f82010-10-27 15:34:10 -0700758 "Anonymous: %8lu kB\n"
Dave Hansen4031a212011-03-22 16:33:01 -0700759 "AnonHugePages: %8lu kB\n"
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800760 "Shared_Hugetlb: %8lu kB\n"
761 "Private_Hugetlb: %7lu kB\n"
Mel Gorman08fba692009-01-06 14:38:53 -0800762 "Swap: %8lu kB\n"
Minchan Kim8334b962015-09-08 15:00:24 -0700763 "SwapPss: %8lu kB\n"
Mel Gorman33402892009-01-06 14:38:54 -0800764 "KernelPageSize: %8lu kB\n"
Nikanth Karthikesan2d905082011-01-13 15:45:53 -0800765 "MMUPageSize: %8lu kB\n"
766 "Locked: %8lu kB\n",
Matt Mackall4752c362008-02-04 22:29:02 -0800767 (vma->vm_end - vma->vm_start) >> 10,
768 mss.resident >> 10,
769 (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
770 mss.shared_clean >> 10,
771 mss.shared_dirty >> 10,
772 mss.private_clean >> 10,
773 mss.private_dirty >> 10,
Peter Zijlstra214e4712008-04-28 02:12:55 -0700774 mss.referenced >> 10,
Nikanth Karthikesanb40d4f82010-10-27 15:34:10 -0700775 mss.anonymous >> 10,
Dave Hansen4031a212011-03-22 16:33:01 -0700776 mss.anonymous_thp >> 10,
Naoya Horiguchi25ee01a2015-11-05 18:47:11 -0800777 mss.shared_hugetlb >> 10,
778 mss.private_hugetlb >> 10,
Mel Gorman08fba692009-01-06 14:38:53 -0800779 mss.swap >> 10,
Minchan Kim8334b962015-09-08 15:00:24 -0700780 (unsigned long)(mss.swap_pss >> (10 + PSS_SHIFT)),
Mel Gorman33402892009-01-06 14:38:54 -0800781 vma_kernel_pagesize(vma) >> 10,
Nikanth Karthikesan2d905082011-01-13 15:45:53 -0800782 vma_mmu_pagesize(vma) >> 10,
783 (vma->vm_flags & VM_LOCKED) ?
784 (unsigned long)(mss.pss >> (10 + PSS_SHIFT)) : 0);
Matt Mackall4752c362008-02-04 22:29:02 -0800785
Cyrill Gorcunov834f82e2012-12-17 16:03:13 -0800786 show_smap_vma_flags(m, vma);
Oleg Nesterovb8c20a92014-10-09 15:25:41 -0700787 m_cache_vma(m, vma);
Joe Korty7c88db02008-10-16 15:27:09 +0400788 return 0;
Mauricio Line070ad42005-09-03 15:55:10 -0700789}
790
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700791static int show_pid_smap(struct seq_file *m, void *v)
792{
793 return show_smap(m, v, 1);
794}
795
796static int show_tid_smap(struct seq_file *m, void *v)
797{
798 return show_smap(m, v, 0);
799}
800
Jan Engelhardt03a44822008-02-08 04:21:19 -0800801static const struct seq_operations proc_pid_smaps_op = {
Matt Mackalla6198792008-02-04 22:29:03 -0800802 .start = m_start,
803 .next = m_next,
804 .stop = m_stop,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700805 .show = show_pid_smap
Matt Mackalla6198792008-02-04 22:29:03 -0800806};
807
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700808static const struct seq_operations proc_tid_smaps_op = {
809 .start = m_start,
810 .next = m_next,
811 .stop = m_stop,
812 .show = show_tid_smap
813};
814
815static int pid_smaps_open(struct inode *inode, struct file *file)
Matt Mackalla6198792008-02-04 22:29:03 -0800816{
817 return do_maps_open(inode, file, &proc_pid_smaps_op);
818}
819
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700820static int tid_smaps_open(struct inode *inode, struct file *file)
821{
822 return do_maps_open(inode, file, &proc_tid_smaps_op);
823}
824
825const struct file_operations proc_pid_smaps_operations = {
826 .open = pid_smaps_open,
827 .read = seq_read,
828 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700829 .release = proc_map_release,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700830};
831
832const struct file_operations proc_tid_smaps_operations = {
833 .open = tid_smaps_open,
Matt Mackalla6198792008-02-04 22:29:03 -0800834 .read = seq_read,
835 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -0700836 .release = proc_map_release,
Matt Mackalla6198792008-02-04 22:29:03 -0800837};
838
Pavel Emelyanov040fa022013-07-03 15:01:16 -0700839enum clear_refs_types {
840 CLEAR_REFS_ALL = 1,
841 CLEAR_REFS_ANON,
842 CLEAR_REFS_MAPPED,
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700843 CLEAR_REFS_SOFT_DIRTY,
Petr Cermak695f0552015-02-12 15:01:00 -0800844 CLEAR_REFS_MM_HIWATER_RSS,
Pavel Emelyanov040fa022013-07-03 15:01:16 -0700845 CLEAR_REFS_LAST,
846};
847
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -0700848struct clear_refs_private {
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700849 enum clear_refs_types type;
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -0700850};
851
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800852#ifdef CONFIG_MEM_SOFT_DIRTY
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700853static inline void clear_soft_dirty(struct vm_area_struct *vma,
854 unsigned long addr, pte_t *pte)
855{
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700856 /*
857 * The soft-dirty tracker uses #PF-s to catch writes
858 * to pages, so write-protect the pte as well. See the
859 * Documentation/vm/soft-dirty.txt for full description
860 * of how soft-dirty works.
861 */
862 pte_t ptent = *pte;
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700863
864 if (pte_present(ptent)) {
Laurent Dufour326c2592015-11-05 18:49:21 -0800865 ptent = ptep_modify_prot_start(vma->vm_mm, addr, pte);
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700866 ptent = pte_wrprotect(ptent);
Martin Schwidefskya7b76172015-04-22 14:20:47 +0200867 ptent = pte_clear_soft_dirty(ptent);
Laurent Dufour326c2592015-11-05 18:49:21 -0800868 ptep_modify_prot_commit(vma->vm_mm, addr, pte, ptent);
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700869 } else if (is_swap_pte(ptent)) {
870 ptent = pte_swp_clear_soft_dirty(ptent);
Laurent Dufour326c2592015-11-05 18:49:21 -0800871 set_pte_at(vma->vm_mm, addr, pte, ptent);
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700872 }
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700873}
Laurent Dufour5d3875a2015-11-05 18:49:24 -0800874#else
875static inline void clear_soft_dirty(struct vm_area_struct *vma,
876 unsigned long addr, pte_t *pte)
877{
878}
879#endif
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700880
Laurent Dufour5d3875a2015-11-05 18:49:24 -0800881#if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800882static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
883 unsigned long addr, pmd_t *pmdp)
884{
Laurent Dufour326c2592015-11-05 18:49:21 -0800885 pmd_t pmd = pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800886
887 pmd = pmd_wrprotect(pmd);
Martin Schwidefskya7b76172015-04-22 14:20:47 +0200888 pmd = pmd_clear_soft_dirty(pmd);
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800889
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800890 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
891}
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800892#else
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800893static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
894 unsigned long addr, pmd_t *pmdp)
895{
896}
897#endif
898
Matt Mackalla6198792008-02-04 22:29:03 -0800899static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
Dave Hansen21650092008-06-12 15:21:47 -0700900 unsigned long end, struct mm_walk *walk)
Matt Mackalla6198792008-02-04 22:29:03 -0800901{
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -0700902 struct clear_refs_private *cp = walk->private;
Naoya Horiguchi5c64f522015-02-11 15:27:46 -0800903 struct vm_area_struct *vma = walk->vma;
Matt Mackalla6198792008-02-04 22:29:03 -0800904 pte_t *pte, ptent;
905 spinlock_t *ptl;
906 struct page *page;
907
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -0800908 ptl = pmd_trans_huge_lock(pmd, vma);
909 if (ptl) {
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800910 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
911 clear_soft_dirty_pmd(vma, addr, pmd);
912 goto out;
913 }
914
915 page = pmd_page(*pmd);
916
917 /* Clear accessed and referenced bits. */
918 pmdp_test_and_clear_young(vma, addr, pmd);
Vladimir Davydov33c3fc72015-09-09 15:35:45 -0700919 test_and_clear_page_young(page);
Kirill A. Shutemov7d5b3bf2015-02-11 15:28:08 -0800920 ClearPageReferenced(page);
921out:
922 spin_unlock(ptl);
923 return 0;
924 }
925
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -0700926 if (pmd_trans_unstable(pmd))
927 return 0;
Dave Hansen03319322011-03-22 16:32:56 -0700928
Matt Mackalla6198792008-02-04 22:29:03 -0800929 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
930 for (; addr != end; pte++, addr += PAGE_SIZE) {
931 ptent = *pte;
Matt Mackalla6198792008-02-04 22:29:03 -0800932
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -0700933 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
934 clear_soft_dirty(vma, addr, pte);
935 continue;
936 }
937
Cyrill Gorcunov179ef712013-08-13 16:00:49 -0700938 if (!pte_present(ptent))
939 continue;
940
Matt Mackalla6198792008-02-04 22:29:03 -0800941 page = vm_normal_page(vma, addr, ptent);
942 if (!page)
943 continue;
944
945 /* Clear accessed and referenced bits. */
946 ptep_test_and_clear_young(vma, addr, pte);
Vladimir Davydov33c3fc72015-09-09 15:35:45 -0700947 test_and_clear_page_young(page);
Matt Mackalla6198792008-02-04 22:29:03 -0800948 ClearPageReferenced(page);
949 }
950 pte_unmap_unlock(pte - 1, ptl);
951 cond_resched();
952 return 0;
953}
954
Naoya Horiguchi5c64f522015-02-11 15:27:46 -0800955static int clear_refs_test_walk(unsigned long start, unsigned long end,
956 struct mm_walk *walk)
957{
958 struct clear_refs_private *cp = walk->private;
959 struct vm_area_struct *vma = walk->vma;
960
Naoya Horiguchi48684a62015-02-11 15:28:06 -0800961 if (vma->vm_flags & VM_PFNMAP)
962 return 1;
963
Naoya Horiguchi5c64f522015-02-11 15:27:46 -0800964 /*
965 * Writing 1 to /proc/pid/clear_refs affects all pages.
966 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
967 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
968 * Writing 4 to /proc/pid/clear_refs affects all pages.
969 */
970 if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
971 return 1;
972 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
973 return 1;
974 return 0;
975}
976
Matt Mackallf248dcb2008-02-04 22:29:03 -0800977static ssize_t clear_refs_write(struct file *file, const char __user *buf,
978 size_t count, loff_t *ppos)
David Rientjesb813e932007-05-06 14:49:24 -0700979{
Matt Mackallf248dcb2008-02-04 22:29:03 -0800980 struct task_struct *task;
Vincent Lifb92a4b2009-09-22 16:45:36 -0700981 char buffer[PROC_NUMBUF];
Matt Mackallf248dcb2008-02-04 22:29:03 -0800982 struct mm_struct *mm;
David Rientjesb813e932007-05-06 14:49:24 -0700983 struct vm_area_struct *vma;
Pavel Emelyanov040fa022013-07-03 15:01:16 -0700984 enum clear_refs_types type;
985 int itype;
Alexey Dobriyan0a8cb8e2011-05-26 16:25:50 -0700986 int rv;
David Rientjesb813e932007-05-06 14:49:24 -0700987
Matt Mackallf248dcb2008-02-04 22:29:03 -0800988 memset(buffer, 0, sizeof(buffer));
989 if (count > sizeof(buffer) - 1)
990 count = sizeof(buffer) - 1;
991 if (copy_from_user(buffer, buf, count))
992 return -EFAULT;
Pavel Emelyanov040fa022013-07-03 15:01:16 -0700993 rv = kstrtoint(strstrip(buffer), 10, &itype);
Alexey Dobriyan0a8cb8e2011-05-26 16:25:50 -0700994 if (rv < 0)
995 return rv;
Pavel Emelyanov040fa022013-07-03 15:01:16 -0700996 type = (enum clear_refs_types)itype;
997 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
Matt Mackallf248dcb2008-02-04 22:29:03 -0800998 return -EINVAL;
Pavel Emelyanov541c2372013-07-03 15:01:22 -0700999
Al Viro496ad9a2013-01-23 17:07:38 -05001000 task = get_proc_task(file_inode(file));
Matt Mackallf248dcb2008-02-04 22:29:03 -08001001 if (!task)
1002 return -ESRCH;
1003 mm = get_task_mm(task);
1004 if (mm) {
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -07001005 struct clear_refs_private cp = {
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001006 .type = type,
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -07001007 };
Andrew Morton20cbc972008-07-05 12:29:05 -07001008 struct mm_walk clear_refs_walk = {
1009 .pmd_entry = clear_refs_pte_range,
Naoya Horiguchi5c64f522015-02-11 15:27:46 -08001010 .test_walk = clear_refs_test_walk,
Andrew Morton20cbc972008-07-05 12:29:05 -07001011 .mm = mm,
Pavel Emelyanovaf9de7e2013-07-03 15:01:18 -07001012 .private = &cp,
Andrew Morton20cbc972008-07-05 12:29:05 -07001013 };
Petr Cermak695f0552015-02-12 15:01:00 -08001014
1015 if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1016 /*
1017 * Writing 5 to /proc/pid/clear_refs resets the peak
1018 * resident set size to this mm's current rss value.
1019 */
1020 down_write(&mm->mmap_sem);
1021 reset_mm_hiwater_rss(mm);
1022 up_write(&mm->mmap_sem);
1023 goto out_mm;
1024 }
1025
Matt Mackallf248dcb2008-02-04 22:29:03 -08001026 down_read(&mm->mmap_sem);
Peter Feiner64e45502014-10-13 15:55:46 -07001027 if (type == CLEAR_REFS_SOFT_DIRTY) {
1028 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1029 if (!(vma->vm_flags & VM_SOFTDIRTY))
1030 continue;
1031 up_read(&mm->mmap_sem);
1032 down_write(&mm->mmap_sem);
1033 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1034 vma->vm_flags &= ~VM_SOFTDIRTY;
1035 vma_set_page_prot(vma);
1036 }
1037 downgrade_write(&mm->mmap_sem);
1038 break;
1039 }
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001040 mmu_notifier_invalidate_range_start(mm, 0, -1);
Peter Feiner64e45502014-10-13 15:55:46 -07001041 }
Naoya Horiguchi5c64f522015-02-11 15:27:46 -08001042 walk_page_range(0, ~0UL, &clear_refs_walk);
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001043 if (type == CLEAR_REFS_SOFT_DIRTY)
1044 mmu_notifier_invalidate_range_end(mm, 0, -1);
Matt Mackallf248dcb2008-02-04 22:29:03 -08001045 flush_tlb_mm(mm);
1046 up_read(&mm->mmap_sem);
Petr Cermak695f0552015-02-12 15:01:00 -08001047out_mm:
Matt Mackallf248dcb2008-02-04 22:29:03 -08001048 mmput(mm);
1049 }
1050 put_task_struct(task);
Vincent Lifb92a4b2009-09-22 16:45:36 -07001051
1052 return count;
David Rientjesb813e932007-05-06 14:49:24 -07001053}
1054
Matt Mackallf248dcb2008-02-04 22:29:03 -08001055const struct file_operations proc_clear_refs_operations = {
1056 .write = clear_refs_write,
Arnd Bergmann6038f372010-08-15 18:52:59 +02001057 .llseek = noop_llseek,
Matt Mackallf248dcb2008-02-04 22:29:03 -08001058};
1059
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001060typedef struct {
1061 u64 pme;
1062} pagemap_entry_t;
1063
Matt Mackall85863e42008-02-04 22:29:04 -08001064struct pagemapread {
yonghua zheng8c829622013-08-13 16:01:03 -07001065 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001066 pagemap_entry_t *buffer;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001067 bool show_pfn;
Matt Mackall85863e42008-02-04 22:29:04 -08001068};
1069
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001070#define PAGEMAP_WALK_SIZE (PMD_SIZE)
1071#define PAGEMAP_WALK_MASK (PMD_MASK)
1072
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001073#define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1074#define PM_PFRAME_BITS 55
1075#define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1076#define PM_SOFT_DIRTY BIT_ULL(55)
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001077#define PM_MMAP_EXCLUSIVE BIT_ULL(56)
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001078#define PM_FILE BIT_ULL(61)
1079#define PM_SWAP BIT_ULL(62)
1080#define PM_PRESENT BIT_ULL(63)
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001081
Matt Mackall85863e42008-02-04 22:29:04 -08001082#define PM_END_OF_BUFFER 1
1083
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001084static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001085{
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001086 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001087}
1088
1089static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
Matt Mackall85863e42008-02-04 22:29:04 -08001090 struct pagemapread *pm)
1091{
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001092 pm->buffer[pm->pos++] = *pme;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001093 if (pm->pos >= pm->len)
Thomas Tuttleaae86792008-06-05 22:46:31 -07001094 return PM_END_OF_BUFFER;
Matt Mackall85863e42008-02-04 22:29:04 -08001095 return 0;
1096}
1097
1098static int pagemap_pte_hole(unsigned long start, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -07001099 struct mm_walk *walk)
Matt Mackall85863e42008-02-04 22:29:04 -08001100{
Dave Hansen21650092008-06-12 15:21:47 -07001101 struct pagemapread *pm = walk->private;
Peter Feiner68b5a652014-08-06 16:08:09 -07001102 unsigned long addr = start;
Matt Mackall85863e42008-02-04 22:29:04 -08001103 int err = 0;
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001104
Peter Feiner68b5a652014-08-06 16:08:09 -07001105 while (addr < end) {
1106 struct vm_area_struct *vma = find_vma(walk->mm, addr);
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001107 pagemap_entry_t pme = make_pme(0, 0);
Peter Feiner87e6d492014-09-25 16:05:18 -07001108 /* End of address space hole, which we mark as non-present. */
1109 unsigned long hole_end;
Peter Feiner68b5a652014-08-06 16:08:09 -07001110
Peter Feiner87e6d492014-09-25 16:05:18 -07001111 if (vma)
1112 hole_end = min(end, vma->vm_start);
1113 else
1114 hole_end = end;
1115
1116 for (; addr < hole_end; addr += PAGE_SIZE) {
1117 err = add_to_pagemap(addr, &pme, pm);
1118 if (err)
1119 goto out;
Peter Feiner68b5a652014-08-06 16:08:09 -07001120 }
1121
Peter Feiner87e6d492014-09-25 16:05:18 -07001122 if (!vma)
1123 break;
1124
1125 /* Addresses in the VMA. */
1126 if (vma->vm_flags & VM_SOFTDIRTY)
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001127 pme = make_pme(0, PM_SOFT_DIRTY);
Peter Feiner87e6d492014-09-25 16:05:18 -07001128 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
Peter Feiner68b5a652014-08-06 16:08:09 -07001129 err = add_to_pagemap(addr, &pme, pm);
1130 if (err)
1131 goto out;
1132 }
Matt Mackall85863e42008-02-04 22:29:04 -08001133 }
Peter Feiner68b5a652014-08-06 16:08:09 -07001134out:
Matt Mackall85863e42008-02-04 22:29:04 -08001135 return err;
1136}
1137
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001138static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001139 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
Matt Mackall85863e42008-02-04 22:29:04 -08001140{
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001141 u64 frame = 0, flags = 0;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001142 struct page *page = NULL;
Matt Mackall85863e42008-02-04 22:29:04 -08001143
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001144 if (pte_present(pte)) {
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001145 if (pm->show_pfn)
1146 frame = pte_pfn(pte);
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001147 flags |= PM_PRESENT;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001148 page = vm_normal_page(vma, addr, pte);
Cyrill Gorcunove9cdd6e2013-10-16 13:46:53 -07001149 if (pte_soft_dirty(pte))
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001150 flags |= PM_SOFT_DIRTY;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001151 } else if (is_swap_pte(pte)) {
Cyrill Gorcunov179ef712013-08-13 16:00:49 -07001152 swp_entry_t entry;
1153 if (pte_swp_soft_dirty(pte))
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001154 flags |= PM_SOFT_DIRTY;
Cyrill Gorcunov179ef712013-08-13 16:00:49 -07001155 entry = pte_to_swp_entry(pte);
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001156 frame = swp_type(entry) |
1157 (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001158 flags |= PM_SWAP;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001159 if (is_migration_entry(entry))
1160 page = migration_entry_to_page(entry);
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001161 }
1162
1163 if (page && !PageAnon(page))
1164 flags |= PM_FILE;
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001165 if (page && page_mapcount(page) == 1)
1166 flags |= PM_MMAP_EXCLUSIVE;
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001167 if (vma->vm_flags & VM_SOFTDIRTY)
1168 flags |= PM_SOFT_DIRTY;
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001169
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001170 return make_pme(frame, flags);
Dave Hansenbcf80392008-06-12 15:21:48 -07001171}
1172
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001173static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
Dave Hansen21650092008-06-12 15:21:47 -07001174 struct mm_walk *walk)
Matt Mackall85863e42008-02-04 22:29:04 -08001175{
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001176 struct vm_area_struct *vma = walk->vma;
Dave Hansen21650092008-06-12 15:21:47 -07001177 struct pagemapread *pm = walk->private;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001178 spinlock_t *ptl;
Konstantin Khlebnikov05fbf352015-02-11 15:27:31 -08001179 pte_t *pte, *orig_pte;
Matt Mackall85863e42008-02-04 22:29:04 -08001180 int err = 0;
1181
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001182#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001183 ptl = pmd_trans_huge_lock(pmdp, vma);
1184 if (ptl) {
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001185 u64 flags = 0, frame = 0;
1186 pmd_t pmd = *pmdp;
Pavel Emelyanov0f8975e2013-07-03 15:01:20 -07001187
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001188 if ((vma->vm_flags & VM_SOFTDIRTY) || pmd_soft_dirty(pmd))
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001189 flags |= PM_SOFT_DIRTY;
Cyrill Gorcunovd9104d12013-09-11 14:22:24 -07001190
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001191 /*
1192 * Currently pmd for thp is always present because thp
1193 * can not be swapped-out, migrated, or HWPOISONed
1194 * (split in such cases instead.)
1195 * This if-check is just to prepare for future implementation.
1196 */
1197 if (pmd_present(pmd)) {
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001198 struct page *page = pmd_page(pmd);
1199
1200 if (page_mapcount(page) == 1)
1201 flags |= PM_MMAP_EXCLUSIVE;
1202
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001203 flags |= PM_PRESENT;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001204 if (pm->show_pfn)
1205 frame = pmd_pfn(pmd) +
1206 ((addr & ~PMD_MASK) >> PAGE_SHIFT);
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001207 }
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001208
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001209 for (; addr != end; addr += PAGE_SIZE) {
1210 pagemap_entry_t pme = make_pme(frame, flags);
1211
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001212 err = add_to_pagemap(addr, &pme, pm);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001213 if (err)
1214 break;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001215 if (pm->show_pfn && (flags & PM_PRESENT))
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001216 frame++;
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001217 }
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001218 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001219 return err;
Naoya Horiguchi5aaabe82012-03-21 16:33:57 -07001220 }
1221
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001222 if (pmd_trans_unstable(pmdp))
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07001223 return 0;
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001224#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
Dave Hansenbcf80392008-06-12 15:21:48 -07001225
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001226 /*
1227 * We can assume that @vma always points to a valid one and @end never
1228 * goes beyond vma->vm_end.
1229 */
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001230 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001231 for (; addr < end; pte++, addr += PAGE_SIZE) {
1232 pagemap_entry_t pme;
Peter Feiner81d0fa62014-10-09 15:28:32 -07001233
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001234 pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001235 err = add_to_pagemap(addr, &pme, pm);
Konstantin Khlebnikov05fbf352015-02-11 15:27:31 -08001236 if (err)
Peter Feiner81d0fa62014-10-09 15:28:32 -07001237 break;
Matt Mackall85863e42008-02-04 22:29:04 -08001238 }
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001239 pte_unmap_unlock(orig_pte, ptl);
Matt Mackall85863e42008-02-04 22:29:04 -08001240
1241 cond_resched();
1242
1243 return err;
1244}
1245
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001246#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi116354d2010-04-06 14:35:04 -07001247/* This function walks within one hugetlb entry in the single call */
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001248static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
Naoya Horiguchi116354d2010-04-06 14:35:04 -07001249 unsigned long addr, unsigned long end,
1250 struct mm_walk *walk)
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001251{
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001252 struct pagemapread *pm = walk->private;
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001253 struct vm_area_struct *vma = walk->vma;
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001254 u64 flags = 0, frame = 0;
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001255 int err = 0;
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001256 pte_t pte;
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001257
Naoya Horiguchif995ece2015-02-11 15:27:48 -08001258 if (vma->vm_flags & VM_SOFTDIRTY)
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001259 flags |= PM_SOFT_DIRTY;
Cyrill Gorcunovd9104d12013-09-11 14:22:24 -07001260
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001261 pte = huge_ptep_get(ptep);
1262 if (pte_present(pte)) {
1263 struct page *page = pte_page(pte);
1264
1265 if (!PageAnon(page))
1266 flags |= PM_FILE;
1267
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001268 if (page_mapcount(page) == 1)
1269 flags |= PM_MMAP_EXCLUSIVE;
1270
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001271 flags |= PM_PRESENT;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001272 if (pm->show_pfn)
1273 frame = pte_pfn(pte) +
1274 ((addr & ~hmask) >> PAGE_SHIFT);
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001275 }
1276
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001277 for (; addr != end; addr += PAGE_SIZE) {
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001278 pagemap_entry_t pme = make_pme(frame, flags);
1279
Naoya Horiguchi092b50b2012-03-21 16:33:59 -07001280 err = add_to_pagemap(addr, &pme, pm);
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001281 if (err)
1282 return err;
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001283 if (pm->show_pfn && (flags & PM_PRESENT))
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001284 frame++;
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001285 }
1286
1287 cond_resched();
1288
1289 return err;
1290}
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001291#endif /* HUGETLB_PAGE */
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001292
Matt Mackall85863e42008-02-04 22:29:04 -08001293/*
1294 * /proc/pid/pagemap - an array mapping virtual pages to pfns
1295 *
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001296 * For each page in the address space, this file contains one 64-bit entry
1297 * consisting of the following:
1298 *
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001299 * Bits 0-54 page frame number (PFN) if present
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001300 * Bits 0-4 swap type if swapped
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001301 * Bits 5-54 swap offset if swapped
Konstantin Khlebnikovdeb94542015-09-08 15:00:02 -07001302 * Bit 55 pte is soft-dirty (see Documentation/vm/soft-dirty.txt)
Konstantin Khlebnikov77bb4992015-09-08 15:00:10 -07001303 * Bit 56 page exclusively mapped
1304 * Bits 57-60 zero
Konstantin Khlebnikov052fb0d2012-05-31 16:26:19 -07001305 * Bit 61 page is file-page or shared-anon
Hans Rosenfeldf16278c2008-03-21 18:46:59 -05001306 * Bit 62 page swapped
1307 * Bit 63 page present
1308 *
1309 * If the page is not present but in swap, then the PFN contains an
1310 * encoding of the swap file number and the page's offset into the
1311 * swap. Unmapped pages return a null PFN. This allows determining
Matt Mackall85863e42008-02-04 22:29:04 -08001312 * precisely which pages are mapped (or in swap) and comparing mapped
1313 * pages between processes.
1314 *
1315 * Efficient users of this interface will use /proc/pid/maps to
1316 * determine which areas of memory are actually mapped and llseek to
1317 * skip over unmapped regions.
1318 */
1319static ssize_t pagemap_read(struct file *file, char __user *buf,
1320 size_t count, loff_t *ppos)
1321{
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001322 struct mm_struct *mm = file->private_data;
Matt Mackall85863e42008-02-04 22:29:04 -08001323 struct pagemapread pm;
Alexey Dobriyanee1e6ab2008-07-21 14:21:36 -07001324 struct mm_walk pagemap_walk = {};
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001325 unsigned long src;
1326 unsigned long svpfn;
1327 unsigned long start_vaddr;
1328 unsigned long end_vaddr;
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001329 int ret = 0, copied = 0;
Matt Mackall85863e42008-02-04 22:29:04 -08001330
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001331 if (!mm || !atomic_inc_not_zero(&mm->mm_users))
Matt Mackall85863e42008-02-04 22:29:04 -08001332 goto out;
1333
Matt Mackall85863e42008-02-04 22:29:04 -08001334 ret = -EINVAL;
1335 /* file position must be aligned */
Thomas Tuttleaae86792008-06-05 22:46:31 -07001336 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001337 goto out_mm;
Matt Mackall85863e42008-02-04 22:29:04 -08001338
1339 ret = 0;
Vitaly Mayatskikh08161782009-04-30 15:08:18 -07001340 if (!count)
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001341 goto out_mm;
Vitaly Mayatskikh08161782009-04-30 15:08:18 -07001342
Konstantin Khlebnikov1c903082015-09-08 15:00:07 -07001343 /* do not disclose physical addresses: attack vector */
1344 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1345
yonghua zheng8c829622013-08-13 16:01:03 -07001346 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1347 pm.buffer = kmalloc(pm.len * PM_ENTRY_BYTES, GFP_TEMPORARY);
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001348 ret = -ENOMEM;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001349 if (!pm.buffer)
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001350 goto out_mm;
Matt Mackall85863e42008-02-04 22:29:04 -08001351
Konstantin Khlebnikov356515e2015-09-08 15:00:04 -07001352 pagemap_walk.pmd_entry = pagemap_pmd_range;
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001353 pagemap_walk.pte_hole = pagemap_pte_hole;
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001354#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001355 pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
Naoya Horiguchi1a5cb812010-05-24 14:32:12 -07001356#endif
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001357 pagemap_walk.mm = mm;
1358 pagemap_walk.private = &pm;
Matt Mackall85863e42008-02-04 22:29:04 -08001359
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001360 src = *ppos;
1361 svpfn = src / PM_ENTRY_BYTES;
1362 start_vaddr = svpfn << PAGE_SHIFT;
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001363 end_vaddr = mm->task_size;
Matt Mackall85863e42008-02-04 22:29:04 -08001364
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001365 /* watch out for wraparound */
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001366 if (svpfn > mm->task_size >> PAGE_SHIFT)
Andrew Morton5d7e0d22008-07-05 01:02:01 -07001367 start_vaddr = end_vaddr;
1368
1369 /*
1370 * The odds are that this will stop walking way
1371 * before end_vaddr, because the length of the
1372 * user buffer is tracked in "pm", and the walk
1373 * will stop when we hit the end of the buffer.
1374 */
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001375 ret = 0;
1376 while (count && (start_vaddr < end_vaddr)) {
1377 int len;
1378 unsigned long end;
Matt Mackall85863e42008-02-04 22:29:04 -08001379
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001380 pm.pos = 0;
Naoya Horiguchiea251c12010-11-24 12:57:13 -08001381 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001382 /* overflow ? */
1383 if (end < start_vaddr || end > end_vaddr)
1384 end = end_vaddr;
1385 down_read(&mm->mmap_sem);
1386 ret = walk_page_range(start_vaddr, end, &pagemap_walk);
1387 up_read(&mm->mmap_sem);
1388 start_vaddr = end;
1389
1390 len = min(count, PM_ENTRY_BYTES * pm.pos);
Dan Carpenter309361e02010-04-06 13:45:39 +03001391 if (copy_to_user(buf, pm.buffer, len)) {
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001392 ret = -EFAULT;
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001393 goto out_free;
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001394 }
1395 copied += len;
1396 buf += len;
1397 count -= len;
Matt Mackall85863e42008-02-04 22:29:04 -08001398 }
KAMEZAWA Hiroyukid82ef022010-04-02 09:11:29 +09001399 *ppos += copied;
1400 if (!ret || ret == PM_END_OF_BUFFER)
1401 ret = copied;
1402
KOSAKI Motohiro98bc93e2011-05-26 16:25:53 -07001403out_free:
1404 kfree(pm.buffer);
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001405out_mm:
1406 mmput(mm);
Matt Mackall85863e42008-02-04 22:29:04 -08001407out:
1408 return ret;
1409}
1410
Pavel Emelyanov541c2372013-07-03 15:01:22 -07001411static int pagemap_open(struct inode *inode, struct file *file)
1412{
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001413 struct mm_struct *mm;
1414
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001415 mm = proc_mem_open(inode, PTRACE_MODE_READ);
1416 if (IS_ERR(mm))
1417 return PTR_ERR(mm);
1418 file->private_data = mm;
1419 return 0;
1420}
1421
1422static int pagemap_release(struct inode *inode, struct file *file)
1423{
1424 struct mm_struct *mm = file->private_data;
1425
1426 if (mm)
1427 mmdrop(mm);
Pavel Emelyanov541c2372013-07-03 15:01:22 -07001428 return 0;
1429}
1430
Matt Mackall85863e42008-02-04 22:29:04 -08001431const struct file_operations proc_pagemap_operations = {
1432 .llseek = mem_lseek, /* borrow this */
1433 .read = pagemap_read,
Pavel Emelyanov541c2372013-07-03 15:01:22 -07001434 .open = pagemap_open,
Konstantin Khlebnikova06db752015-09-08 14:59:59 -07001435 .release = pagemap_release,
Matt Mackall85863e42008-02-04 22:29:04 -08001436};
Matt Mackall1e883282008-02-04 22:29:07 -08001437#endif /* CONFIG_PROC_PAGE_MONITOR */
Matt Mackall85863e42008-02-04 22:29:04 -08001438
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001439#ifdef CONFIG_NUMA
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001440
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001441struct numa_maps {
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001442 unsigned long pages;
1443 unsigned long anon;
1444 unsigned long active;
1445 unsigned long writeback;
1446 unsigned long mapcount_max;
1447 unsigned long dirty;
1448 unsigned long swapcache;
1449 unsigned long node[MAX_NUMNODES];
1450};
1451
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001452struct numa_maps_private {
1453 struct proc_maps_private proc_maps;
1454 struct numa_maps md;
1455};
1456
Dave Hanseneb4866d2011-09-20 15:19:38 -07001457static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1458 unsigned long nr_pages)
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001459{
1460 int count = page_mapcount(page);
1461
Dave Hanseneb4866d2011-09-20 15:19:38 -07001462 md->pages += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001463 if (pte_dirty || PageDirty(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001464 md->dirty += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001465
1466 if (PageSwapCache(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001467 md->swapcache += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001468
1469 if (PageActive(page) || PageUnevictable(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001470 md->active += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001471
1472 if (PageWriteback(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001473 md->writeback += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001474
1475 if (PageAnon(page))
Dave Hanseneb4866d2011-09-20 15:19:38 -07001476 md->anon += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001477
1478 if (count > md->mapcount_max)
1479 md->mapcount_max = count;
1480
Dave Hanseneb4866d2011-09-20 15:19:38 -07001481 md->node[page_to_nid(page)] += nr_pages;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001482}
1483
Dave Hansen3200a8a2011-09-20 15:19:39 -07001484static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1485 unsigned long addr)
1486{
1487 struct page *page;
1488 int nid;
1489
1490 if (!pte_present(pte))
1491 return NULL;
1492
1493 page = vm_normal_page(vma, addr, pte);
1494 if (!page)
1495 return NULL;
1496
1497 if (PageReserved(page))
1498 return NULL;
1499
1500 nid = page_to_nid(page);
Lai Jiangshan4ff1b2c2012-12-12 13:51:25 -08001501 if (!node_isset(nid, node_states[N_MEMORY]))
Dave Hansen3200a8a2011-09-20 15:19:39 -07001502 return NULL;
1503
1504 return page;
1505}
1506
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001507static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1508 unsigned long end, struct mm_walk *walk)
1509{
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001510 struct numa_maps *md = walk->private;
1511 struct vm_area_struct *vma = walk->vma;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001512 spinlock_t *ptl;
1513 pte_t *orig_pte;
1514 pte_t *pte;
1515
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001516 ptl = pmd_trans_huge_lock(pmd, vma);
1517 if (ptl) {
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001518 pte_t huge_pte = *(pte_t *)pmd;
1519 struct page *page;
1520
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001521 page = can_gather_numa_stats(huge_pte, vma, addr);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001522 if (page)
1523 gather_stats(page, md, pte_dirty(huge_pte),
1524 HPAGE_PMD_SIZE/PAGE_SIZE);
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001525 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001526 return 0;
Dave Hansen32ef4382011-09-20 15:19:41 -07001527 }
1528
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07001529 if (pmd_trans_unstable(pmd))
1530 return 0;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001531 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1532 do {
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001533 struct page *page = can_gather_numa_stats(*pte, vma, addr);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001534 if (!page)
1535 continue;
Dave Hanseneb4866d2011-09-20 15:19:38 -07001536 gather_stats(page, md, pte_dirty(*pte), 1);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001537
1538 } while (pte++, addr += PAGE_SIZE, addr != end);
1539 pte_unmap_unlock(orig_pte, ptl);
1540 return 0;
1541}
1542#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi632fd602015-02-11 15:27:51 -08001543static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001544 unsigned long addr, unsigned long end, struct mm_walk *walk)
1545{
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001546 pte_t huge_pte = huge_ptep_get(pte);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001547 struct numa_maps *md;
1548 struct page *page;
1549
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001550 if (!pte_present(huge_pte))
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001551 return 0;
1552
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001553 page = pte_page(huge_pte);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001554 if (!page)
1555 return 0;
1556
1557 md = walk->private;
Michael Holzheu5c2ff952016-02-02 16:57:26 -08001558 gather_stats(page, md, pte_dirty(huge_pte), 1);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001559 return 0;
1560}
1561
1562#else
Naoya Horiguchi632fd602015-02-11 15:27:51 -08001563static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001564 unsigned long addr, unsigned long end, struct mm_walk *walk)
1565{
1566 return 0;
1567}
1568#endif
1569
1570/*
1571 * Display pages allocated per node and memory policy via /proc.
1572 */
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001573static int show_numa_map(struct seq_file *m, void *v, int is_pid)
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001574{
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001575 struct numa_maps_private *numa_priv = m->private;
1576 struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001577 struct vm_area_struct *vma = v;
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001578 struct numa_maps *md = &numa_priv->md;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001579 struct file *file = vma->vm_file;
1580 struct mm_struct *mm = vma->vm_mm;
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001581 struct mm_walk walk = {
1582 .hugetlb_entry = gather_hugetlb_stats,
1583 .pmd_entry = gather_pte_stats,
1584 .private = md,
1585 .mm = mm,
1586 };
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001587 struct mempolicy *pol;
David Rientjes948927e2013-11-12 15:07:28 -08001588 char buffer[64];
1589 int nid;
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001590
1591 if (!mm)
1592 return 0;
1593
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001594 /* Ensure we start with an empty set of numa_maps statistics. */
1595 memset(md, 0, sizeof(*md));
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001596
Oleg Nesterov498f2372014-10-09 15:27:52 -07001597 pol = __get_vma_policy(vma, vma->vm_start);
1598 if (pol) {
1599 mpol_to_str(buffer, sizeof(buffer), pol);
1600 mpol_cond_put(pol);
1601 } else {
1602 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1603 }
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001604
1605 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1606
1607 if (file) {
Fabian Frederick17c2b4e2014-06-06 14:37:03 -07001608 seq_puts(m, " file=");
Miklos Szeredi2726d562015-06-19 10:30:28 +02001609 seq_file_path(m, file, "\n\t= ");
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001610 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
Fabian Frederick17c2b4e2014-06-06 14:37:03 -07001611 seq_puts(m, " heap");
Johannes Weiner65376df2016-02-02 16:57:29 -08001612 } else if (is_stack(proc_priv, vma, is_pid)) {
1613 seq_puts(m, " stack");
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001614 }
1615
Andrew Mortonfc360bd2011-10-31 17:06:32 -07001616 if (is_vm_hugetlb_page(vma))
Fabian Frederick17c2b4e2014-06-06 14:37:03 -07001617 seq_puts(m, " huge");
Andrew Mortonfc360bd2011-10-31 17:06:32 -07001618
Naoya Horiguchid85f4d62015-02-11 15:27:54 -08001619 /* mmap_sem is held by m_start */
1620 walk_page_vma(vma, &walk);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001621
1622 if (!md->pages)
1623 goto out;
1624
1625 if (md->anon)
1626 seq_printf(m, " anon=%lu", md->anon);
1627
1628 if (md->dirty)
1629 seq_printf(m, " dirty=%lu", md->dirty);
1630
1631 if (md->pages != md->anon && md->pages != md->dirty)
1632 seq_printf(m, " mapped=%lu", md->pages);
1633
1634 if (md->mapcount_max > 1)
1635 seq_printf(m, " mapmax=%lu", md->mapcount_max);
1636
1637 if (md->swapcache)
1638 seq_printf(m, " swapcache=%lu", md->swapcache);
1639
1640 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1641 seq_printf(m, " active=%lu", md->active);
1642
1643 if (md->writeback)
1644 seq_printf(m, " writeback=%lu", md->writeback);
1645
David Rientjes948927e2013-11-12 15:07:28 -08001646 for_each_node_state(nid, N_MEMORY)
1647 if (md->node[nid])
1648 seq_printf(m, " N%d=%lu", nid, md->node[nid]);
Rafael Aquini198d1592015-02-12 15:01:08 -08001649
1650 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001651out:
1652 seq_putc(m, '\n');
Oleg Nesterovb8c20a92014-10-09 15:25:41 -07001653 m_cache_vma(m, vma);
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001654 return 0;
1655}
Stephen Wilson5b52fc82011-05-24 17:12:49 -07001656
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001657static int show_pid_numa_map(struct seq_file *m, void *v)
1658{
1659 return show_numa_map(m, v, 1);
1660}
1661
1662static int show_tid_numa_map(struct seq_file *m, void *v)
1663{
1664 return show_numa_map(m, v, 0);
1665}
1666
Jan Engelhardt03a44822008-02-08 04:21:19 -08001667static const struct seq_operations proc_pid_numa_maps_op = {
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001668 .start = m_start,
1669 .next = m_next,
1670 .stop = m_stop,
1671 .show = show_pid_numa_map,
Christoph Lameter6e21c8f2005-09-03 15:54:45 -07001672};
Eric W. Biederman662795d2006-06-26 00:25:48 -07001673
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001674static const struct seq_operations proc_tid_numa_maps_op = {
1675 .start = m_start,
1676 .next = m_next,
1677 .stop = m_stop,
1678 .show = show_tid_numa_map,
1679};
1680
1681static int numa_maps_open(struct inode *inode, struct file *file,
1682 const struct seq_operations *ops)
Eric W. Biederman662795d2006-06-26 00:25:48 -07001683{
Oleg Nesterov4db7d0e2014-10-09 15:25:21 -07001684 return proc_maps_open(inode, file, ops,
1685 sizeof(struct numa_maps_private));
Eric W. Biederman662795d2006-06-26 00:25:48 -07001686}
1687
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001688static int pid_numa_maps_open(struct inode *inode, struct file *file)
1689{
1690 return numa_maps_open(inode, file, &proc_pid_numa_maps_op);
1691}
1692
1693static int tid_numa_maps_open(struct inode *inode, struct file *file)
1694{
1695 return numa_maps_open(inode, file, &proc_tid_numa_maps_op);
1696}
1697
1698const struct file_operations proc_pid_numa_maps_operations = {
1699 .open = pid_numa_maps_open,
1700 .read = seq_read,
1701 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -07001702 .release = proc_map_release,
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001703};
1704
1705const struct file_operations proc_tid_numa_maps_operations = {
1706 .open = tid_numa_maps_open,
Eric W. Biederman662795d2006-06-26 00:25:48 -07001707 .read = seq_read,
1708 .llseek = seq_lseek,
Oleg Nesterov29a40ac2014-10-09 15:25:26 -07001709 .release = proc_map_release,
Eric W. Biederman662795d2006-06-26 00:25:48 -07001710};
Stephen Wilsonf69ff942011-05-24 17:12:47 -07001711#endif /* CONFIG_NUMA */